From 5ba9515e03c23077e75c67bd53918b89220561eb Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 09:15:12 +0000 Subject: [PATCH 01/44] feat(server): establish Kilo SDK session boundary --- apps/server/package.json | 1 + .../kilo/KiloSessionClient.live.test.ts | 271 +++++++++++ .../provider/kilo/KiloSessionClient.test.ts | 452 ++++++++++++++++++ .../src/provider/kilo/KiloSessionClient.ts | 409 ++++++++++++++++ pnpm-lock.yaml | 10 + 5 files changed, 1143 insertions(+) create mode 100644 apps/server/src/provider/kilo/KiloSessionClient.live.test.ts create mode 100644 apps/server/src/provider/kilo/KiloSessionClient.test.ts create mode 100644 apps/server/src/provider/kilo/KiloSessionClient.ts diff --git a/apps/server/package.json b/apps/server/package.json index 1740815918ad..68538d3ca1f8 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -33,6 +33,7 @@ "@effect/platform-node": "catalog:", "@effect/platform-node-shared": "catalog:", "@ff-labs/fff-node": "0.9.4", + "@kilocode/sdk": "7.8.3", "@napi-rs/keyring": "^1.3.0", "@opencode-ai/sdk": "^1.3.15", "@opencode/client": "2.0.18", diff --git a/apps/server/src/provider/kilo/KiloSessionClient.live.test.ts b/apps/server/src/provider/kilo/KiloSessionClient.live.test.ts new file mode 100644 index 000000000000..38f2b5387cf7 --- /dev/null +++ b/apps/server/src/provider/kilo/KiloSessionClient.live.test.ts @@ -0,0 +1,271 @@ +// @effect-diagnostics globalTimers:off - bounds a native process startup; assertions wait on events, never sleeps. +// @effect-diagnostics nodeBuiltinImport:off - exercises the real SDK over Node HTTP and native CLI processes. +/** KILO_BIN=/path/to/kilo vp test run . No model or cloud task is run. */ +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeEvents from "node:events"; +import * as NodeHttp from "node:http"; +import { createKiloClient } from "@kilocode/sdk/v2"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { afterAll, beforeAll, describe, expect, it } from "vite-plus/test"; + +import * as KiloSessionClient from "./KiloSessionClient.ts"; + +const binary = process.env.KILO_BIN; +const run = Effect.runPromise; + +describe.runIf(binary !== undefined)("Kilo 7.8.3 native local sessions", () => { + let root: string; + let url: string; + let platform: string; + let child: NodeChildProcess.ChildProcess | undefined; + let exited: Promise | undefined; + let inferenceRequests = 0; + const inference = NodeHttp.createServer((_req, res) => { + inferenceRequests++; + res.writeHead(500); + res.end(); + }); + + beforeAll(async () => { + platform = await run(HostProcessPlatform); + root = await NodeFSP.mkdtemp( + NodePath.join(process.env.KILO_TEST_ROOT ?? NodeOS.tmpdir(), "t3-kilo-"), + ); + await Promise.all( + ["a", "b", "config", "data", "cache", "state"].map((p) => + NodeFSP.mkdir(NodePath.join(root, p)), + ), + ); + inference.listen(0, "127.0.0.1"); + await NodeEvents.EventEmitter.once(inference, "listening"); + const address = inference.address(); + if (address === null || typeof address === "string") + throw new Error("No inference fixture listener"); + child = NodeChildProcess.spawn(binary!, ["serve", "--hostname=127.0.0.1", "--port=0"], { + cwd: root, + detached: platform !== "win32", + env: { + PATH: process.env.PATH, + HTTP_PROXY: process.env.HTTP_PROXY, + HTTPS_PROXY: process.env.HTTPS_PROXY, + NO_PROXY: process.env.NO_PROXY, + NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS, + SSL_CERT_FILE: process.env.SSL_CERT_FILE, + XDG_CONFIG_HOME: NodePath.join(root, "config"), + XDG_DATA_HOME: NodePath.join(root, "data"), + XDG_CACHE_HOME: NodePath.join(root, "cache"), + XDG_STATE_HOME: NodePath.join(root, "state"), + KILO_SERVER_PASSWORD: "local-test-only", + KILO_DISABLE_AUTOUPDATE: "1", + KILO_DISABLE_MODELS_FETCH: "1", + KILO_DISABLE_DEFAULT_PLUGINS: "1", + KILO_DISABLE_EXTERNAL_SKILLS: "1", + KILO_DISABLE_PROJECT_CONFIG: "1", + KILO_CONFIG_CONTENT: JSON.stringify({ + plugin: [], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + name: "Offline fixture", + options: { baseURL: `http://127.0.0.1:${address.port}/v1` }, + models: { test: { name: "Offline", limit: { context: 10000, output: 1000 } } }, + }, + }, + }), + }, + stdio: ["ignore", "pipe", "pipe"], + }); + exited = NodeEvents.EventEmitter.once(child, "exit"); + url = await new Promise((resolve, reject) => { + let output = ""; + const timeout = setTimeout(() => reject(new Error("Kilo startup timed out")), 25000); + child!.on("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child!.on("exit", (code) => { + clearTimeout(timeout); + reject(new Error(`Kilo exited: ${code}`)); + }); + child!.stderr!.on("data", () => {}); + child!.stdout!.on("data", (chunk) => { + output = (output + String(chunk)).slice(-65536); + const match = /kilo server listening on (http:\/\/\S+)/.exec(output); + if (match) { + clearTimeout(timeout); + resolve(match[1]!); + } + }); + }); + }, 30000); + + afterAll(async () => { + if (child?.pid !== undefined && child.exitCode === null) { + if (platform === "win32") child.kill("SIGKILL"); + else process.kill(-child.pid, "SIGKILL"); + await exited; + } + inference.closeAllConnections(); + await new Promise((resolve) => inference.close(() => resolve())); + if (root) await NodeFSP.rm(root, { recursive: true, force: true }); + }); + + const connect = (name: "a" | "b") => + run( + KiloSessionClient.make({ + instanceId: `instance-${name}`, + directory: NodePath.join(root, name), + baseUrl: url, + serverPassword: "local-test-only", + }), + ); + + it("creates concurrent native sessions, reads saved history, resumes and forks without inference", async () => { + const [a, b] = await Promise.all([connect("a"), connect("b")]); + const [ra, rb] = await Promise.all([run(a.create([])), run(b.create([]))]); + expect(ra.sessionId).not.toBe(rb.sessionId); + expect((await run(a.read(ra))).directory).toBe(NodePath.join(root, "a")); + expect((await run(b.read(rb))).directory).toBe(NodePath.join(root, "b")); + const native = createKiloClient({ + baseUrl: url, + directory: NodePath.join(root, "a"), + throwOnError: true, + headers: { Authorization: `Basic ${Buffer.from("kilo:local-test-only").toString("base64")}` }, + }); + // noReply is implemented before the model loop in Kilo 7.8.3 SessionPrompt.prompt. + // Synchronous admission creates real stored messages without any inference calls. + const first = await native.session.prompt({ + sessionID: ra.sessionId, + noReply: true, + model: { providerID: "fixture", modelID: "test" }, + parts: [{ type: "text", text: "first" }], + }); + const second = await native.session.prompt({ + sessionID: ra.sessionId, + noReply: true, + model: { providerID: "fixture", modelID: "test" }, + parts: [{ type: "text", text: "second" }], + }); + const resumed = await connect("a"); + expect((await run(resumed.history(ra))).map((m) => m.info.id)).toEqual([ + first.data!.info.id, + second.data!.info.id, + ]); + expect(await run(b.history(rb))).toEqual([]); + const fork = await run(resumed.fork(ra, second.data!.info.id)); + expect( + (await run(resumed.history(fork))).map((m) => + m.parts.filter((p) => p.type === "text").map((p) => p.text), + ), + ).toEqual([["first"]]); + await run(resumed.revert(ra, second.data!.info.id)); + expect((await run(resumed.read(ra))).revert?.messageID).toBe(second.data!.info.id); + await run(resumed.abort(ra)); + expect((await run(b.read(rb))).id).toBe(rb.sessionId); + expect(inferenceRequests).toBe(0); + }, 30000); + + it("streams actual native message events past sync envelopes without inference", async () => { + const subscribed = Promise.withResolvers(); + // Observe the upstream response to establish stream readiness without timer polling. + const proxy = NodeHttp.createServer((req, res) => { + const upstream = NodeHttp.request( + new URL(req.url!, url), + { + method: req.method, + headers: req.headers, + }, + (response) => { + res.writeHead(response.statusCode!, response.headers); + response.pipe(res); + if (req.url!.startsWith("/event")) subscribed.resolve(); + }, + ); + upstream.on("error", () => res.destroy()); + res.on("close", () => upstream.destroy()); + req.pipe(upstream); + }); + proxy.listen(0, "127.0.0.1"); + await NodeEvents.EventEmitter.once(proxy, "listening"); + const address = proxy.address(); + if (address === null || typeof address === "string") throw new Error("No proxy listener"); + const controller = new AbortController(); + try { + const directory = NodePath.join(root, "a"); + const client = await run( + KiloSessionClient.make({ + instanceId: "native-stream", + directory, + baseUrl: `http://127.0.0.1:${address.port}`, + serverPassword: "local-test-only", + }), + ); + const ref = await run(client.create([])); + const received = run( + client.events(ref).pipe( + Stream.filter((event) => event.type === "message.part.updated"), + Stream.take(1), + Stream.runCollect, + Effect.scoped, + ), + { signal: controller.signal }, + ); + // Always install a rejection handler before the independent native request. + const settled = received.then( + (events) => ({ events }), + (error: unknown) => ({ error }), + ); + await subscribed.promise; + const native = createKiloClient({ + baseUrl: url, + directory, + throwOnError: true, + headers: { + Authorization: `Basic ${Buffer.from("kilo:local-test-only").toString("base64")}`, + }, + }); + const message = await native.session.prompt({ + sessionID: ref.sessionId, + noReply: true, + model: { providerID: "fixture", modelID: "test" }, + parts: [{ type: "text", text: "native stream" }], + }); + const result = await settled; + if ("error" in result) throw result.error; + expect(result.events).toHaveLength(1); + const event = result.events[0]!; + expect(event.type).toBe("message.part.updated"); + if (event.type === "message.part.updated") { + expect(event.properties.part.sessionID).toBe(ref.sessionId); + expect(event.properties.part.messageID).toBe(message.data!.info.id); + } + expect(inferenceRequests).toBe(0); + } finally { + controller.abort(); + proxy.closeAllConnections(); + await new Promise((resolve) => proxy.close(() => resolve())); + } + }, 15000); + + it("rejects wrong auth and foreign native ids using the actual server", async () => { + const error = await run( + KiloSessionClient.make({ + instanceId: "bad", + directory: root, + baseUrl: url, + serverPassword: "wrong", + }).pipe(Effect.flip), + ); + expect(error.reason).toBe("request_failed"); + const [a, b] = await Promise.all([connect("a"), connect("b")]); + const foreign = await run(b.create([])); + const forged = { ...foreign, instanceId: "instance-a", directory: NodePath.join(root, "a") }; + expect((await run(a.abort(forged).pipe(Effect.flip))).reason).toBe("wrong_owner"); + expect((await run(b.read(foreign))).id).toBe(foreign.sessionId); + }, 15000); +}); diff --git a/apps/server/src/provider/kilo/KiloSessionClient.test.ts b/apps/server/src/provider/kilo/KiloSessionClient.test.ts new file mode 100644 index 000000000000..9c4c8853d87b --- /dev/null +++ b/apps/server/src/provider/kilo/KiloSessionClient.test.ts @@ -0,0 +1,452 @@ +// @effect-diagnostics nodeBuiltinImport:off - exercises the real SDK over Node HTTP and native CLI processes. +import * as NodeHttp from "node:http"; +import * as NodeEvents from "node:events"; +import { afterEach, describe, expect, it } from "vite-plus/test"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; +import * as Fiber from "effect/Fiber"; +import { it as effectIt } from "@effect/vitest"; + +import * as KiloSessionClient from "./KiloSessionClient.ts"; + +const cleanups: Array<() => Promise> = []; +afterEach(async () => { + for (const cleanup of cleanups.splice(0)) await cleanup(); +}); + +async function server( + handler: (req: NodeHttp.IncomingMessage, res: NodeHttp.ServerResponse) => void, +) { + const http = NodeHttp.createServer(handler); + http.listen(0, "127.0.0.1"); + await NodeEvents.EventEmitter.once(http, "listening"); + cleanups.push(async () => { + http.closeAllConnections(); + await new Promise((resolve) => http.close(() => resolve())); + }); + const address = http.address(); + if (address === null || typeof address === "string") throw new Error("No test listener"); + return `http://127.0.0.1:${address.port}`; +} +function json(res: NodeHttp.ServerResponse, value: unknown) { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify(value)); +} +const directory = "/work/space #λ"; +const ref = { instanceId: "instance-a", directory, sessionId: "ses_one" }; +const run = Effect.runPromise; + +async function withClient( + handler: (req: NodeHttp.IncomingMessage, res: NodeHttp.ServerResponse) => void, +) { + const baseUrl = await server((req, res) => { + if (req.url?.startsWith("/global/health")) { + json(res, { healthy: true, version: "7.8.3" }); + return; + } + handler(req, res); + }); + return run( + KiloSessionClient.make({ + instanceId: "instance-a", + directory, + baseUrl, + serverPassword: "test-password", + }), + ); +} + +describe("Kilo native SDK boundary", () => { + it("uses Kilo auth and lossless directory routing with the real SDK", async () => { + const requests: Array<{ + url: string; + authorization: string | undefined; + legacyHeader: string | undefined; + }> = []; + const client = await withClient((req, res) => { + requests.push({ + url: req.url!, + authorization: req.headers.authorization, + legacyHeader: req.headers["x-opencode-directory"] as string | undefined, + }); + json(res, { id: ref.sessionId, directory }); + }); + await run(client.read(ref)); + expect(requests).toEqual([ + { + url: `/session/ses_one?directory=${encodeURIComponent(directory).replace(/%20/g, "+")}`, + authorization: `Basic ${Buffer.from("kilo:test-password").toString("base64")}`, + legacyHeader: undefined, + }, + ]); + }); + + it("rejects an unsupported server version through its health response", async () => { + const baseUrl = await server((_req, res) => json(res, { healthy: true, version: "7.9.0" })); + const error = await run( + KiloSessionClient.make({ instanceId: ref.instanceId, directory, baseUrl }).pipe(Effect.flip), + ); + expect(error.reason).toBe("unsupported_version"); + }); + + it("rejects cross-instance refs before sending any request", async () => { + let requests = 0; + const client = await withClient((_req, res) => { + requests++; + json(res, {}); + }); + const error = await run(client.abort({ ...ref, instanceId: "instance-b" }).pipe(Effect.flip)); + expect(error.reason).toBe("wrong_owner"); + expect(requests).toBe(0); + }); + + it("does not treat a directory header as proof that a native session belongs to it", async () => { + const paths: string[] = []; + const client = await withClient((req, res) => { + paths.push(req.url!); + json(res, { id: ref.sessionId, directory: "/other-checkout" }); + }); + const error = await run(client.abort(ref).pipe(Effect.flip)); + expect(error.reason).toBe("wrong_owner"); + expect(paths).toHaveLength(1); + expect(paths[0]).not.toContain("abort"); + }); + + it("accepts native 204 prompt admission and sends a mutation only once", async () => { + const prompts: unknown[] = []; + const client = await withClient((req, res) => { + if (req.method === "GET") { + json(res, { id: ref.sessionId, directory }); + return; + } + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", () => { + prompts.push(JSON.parse(body)); + res.writeHead(204); + res.end(); + }); + }); + await run( + client.prompt(ref, { + parts: [{ type: "text", text: "one prompt" }], + model: { providerID: "local", modelID: "test" }, + }), + ); + expect(prompts).toEqual([ + { + parts: [{ type: "text", text: "one prompt" }], + model: { providerID: "local", modelID: "test" }, + }, + ]); + }); + + it("does not retry a prompt after the server accepts it but loses the response", async () => { + let admissions = 0; + const client = await withClient((req, res) => { + if (req.method === "GET") { + json(res, { id: ref.sessionId, directory }); + return; + } + admissions++; + req.resume(); + req.on("end", () => res.destroy()); + }); + const error = await run( + client.prompt(ref, { parts: [{ type: "text", text: "one prompt" }] }).pipe(Effect.flip), + ); + expect(error.reason).toBe("admission_unknown"); + expect(admissions).toBe(1); + }); + + it("does not answer another session's approval even when its request id is known", async () => { + let replies = 0; + const client = await withClient((req, res) => { + if (req.method === "POST") replies++; + json( + res, + req.url!.startsWith("/permission") + ? [ + { + id: "request-one", + sessionID: "ses_other", + permission: "bash", + patterns: ["*"], + metadata: {}, + always: [], + }, + ] + : { id: ref.sessionId, directory }, + ); + }); + const error = await run(client.replyPermission(ref, "request-one", "once").pipe(Effect.flip)); + expect(error.reason).toBe("wrong_owner"); + expect(replies).toBe(0); + }); + + it("does not answer another session's question", async () => { + let replies = 0; + const client = await withClient((req, res) => { + if (req.method === "POST") replies++; + json( + res, + req.url!.startsWith("/question") + ? [{ id: "question-one", sessionID: "ses_other", questions: [] }] + : { id: ref.sessionId, directory }, + ); + }); + const error = await run(client.replyQuestion(ref, "question-one", [["yes"]]).pipe(Effect.flip)); + expect(error.reason).toBe("wrong_owner"); + expect(replies).toBe(0); + }); + + it("filters interleaved events and reports EOF without reconnecting or claiming completion", async () => { + let subscriptions = 0; + const events: string[] = []; + const client = await withClient((req, res) => { + if (!req.url!.startsWith("/event")) { + json(res, { id: ref.sessionId, directory }); + return; + } + subscriptions++; + res.writeHead(200, { "Content-Type": "text/event-stream" }); + res.write( + `data: ${JSON.stringify({ type: "sync", syncEvent: { type: "session.updated.v1", id: "sync_one", data: {} } })}\n\n`, + ); + res.write( + `data: ${JSON.stringify({ type: "pty.created", properties: { info: { id: "pty_one", title: "shell", command: "bash", args: [], cwd: directory, status: "running", pid: 42 } } })}\n\n`, + ); + for (const sessionID of ["ses_other", ref.sessionId, "ses_other", ref.sessionId]) { + res.write( + `data: ${JSON.stringify({ type: "message.part.delta", properties: { sessionID, messageID: "same-message", partID: "same-part", field: "text", delta: sessionID } })}\n\n`, + ); + } + res.end(); + }); + const error = await run( + client.events(ref).pipe( + Stream.runForEach((event) => + Effect.sync(() => { + if (event.type === "message.part.delta") events.push(event.properties.delta); + }), + ), + Effect.scoped, + Effect.flip, + ), + ); + expect(events).toEqual([ref.sessionId, ref.sessionId]); + expect(error.reason).toBe("request_failed"); + expect(subscriptions).toBe(1); + }); + + it("refuses HTTP redirects before forwarding a prompt or credentials", async () => { + let foreignRequests = 0; + const other = await server((_req, res) => { + foreignRequests++; + json(res, {}); + }); + const client = await withClient((_req, res) => { + res.writeHead(307, { Location: other }); + res.end(); + }); + const error = await run(client.read(ref).pipe(Effect.flip)); + expect(error.reason).toBe("request_failed"); + expect(foreignRequests).toBe(0); + }); + + it("does not turn a rejected abort into a confirmed stop", async () => { + const client = await withClient((req, res) => + json(res, req.method === "POST" ? false : { id: ref.sessionId, directory }), + ); + expect((await run(client.abort(ref).pipe(Effect.flip))).reason).toBe("invalid_response"); + }); + it.each([null, {}, { success: false }, "true"])( + "rejects a malformed abort acknowledgement %j", + async (value) => { + const client = await withClient((req, res) => + json(res, req.method === "POST" ? value : { id: ref.sessionId, directory }), + ); + expect((await run(client.abort(ref).pipe(Effect.flip))).reason).toBe("invalid_response"); + }, + ); + + it("rejects an empty abort acknowledgement", async () => { + const client = await withClient((req, res) => { + if (req.method === "POST") { + res.writeHead(204, { "Content-Type": "application/json" }); + res.end(); + } else json(res, { id: ref.sessionId, directory }); + }); + expect((await run(client.abort(ref).pipe(Effect.flip))).reason).toBe("invalid_response"); + }); + + it.each([ + null, + { type: "message.part.delta", properties: null }, + { type: "message.part.updated", properties: { part: null } }, + ])("turns malformed SSE into a typed failure %j", async (event) => { + const client = await withClient((req, res) => { + if (!req.url!.startsWith("/event")) { + json(res, { id: ref.sessionId, directory }); + return; + } + res.writeHead(200, { "Content-Type": "text/event-stream" }); + res.end(`data: ${JSON.stringify(event)}\n\n`); + }); + expect( + (await run(client.events(ref).pipe(Stream.runDrain, Effect.scoped, Effect.flip))).reason, + ).toBe("invalid_response"); + }); + + it("closes the actual SSE socket when its consumer is cancelled", async () => { + let close!: () => void; + const closed = new Promise((resolve) => { + close = resolve; + }); + const client = await withClient((req, res) => { + if (!req.url!.startsWith("/event")) { + json(res, { id: ref.sessionId, directory }); + return; + } + res.on("close", close); + res.writeHead(200, { "Content-Type": "text/event-stream" }); + res.write( + `data: ${JSON.stringify({ type: "session.idle", properties: { sessionID: ref.sessionId } })}\n\n`, + ); + }); + await run(client.events(ref).pipe(Stream.take(1), Stream.runDrain, Effect.scoped)); + await closed; + }); + + effectIt.effect("interrupts a blocked SSE read before waiting for iterator cleanup", () => + Effect.gen(function* () { + const opened = Promise.withResolvers(); + const closed = Promise.withResolvers(); + const client = yield* Effect.promise(() => + withClient((req, res) => { + if (!req.url!.startsWith("/event")) { + json(res, { id: ref.sessionId, directory }); + return; + } + res.on("close", closed.resolve); + res.writeHead(200, { "Content-Type": "text/event-stream" }); + res.flushHeaders(); + opened.resolve(); + }), + ); + const consumer = yield* client + .events(ref) + .pipe(Stream.runDrain, Effect.scoped, Effect.forkScoped); + yield* Effect.promise(() => opened.promise); + yield* Fiber.interrupt(consumer); + yield* Effect.promise(() => closed.promise); + }).pipe(Effect.scoped), + ); + + it("refuses a mutation redirect after a successful ownership read", async () => { + let forwarded = 0; + const foreign = await server((_req, res) => { + forwarded++; + json(res, {}); + }); + const client = await withClient((req, res) => { + if (req.method === "GET") { + json(res, { id: ref.sessionId, directory }); + return; + } + res.writeHead(307, { Location: foreign }); + res.end(); + }); + expect((await run(client.prompt(ref, { parts: [] }).pipe(Effect.flip))).reason).toBe( + "admission_unknown", + ); + expect(forwarded).toBe(0); + }); + it("keeps simultaneous client streams separate even with identical native item ids", async () => { + const streams: Array<{ res: NodeHttp.ServerResponse; sessionId: string }> = []; + const baseUrl = await server((req, res) => { + const url = new URL(req.url!, "http://localhost"); + if (url.pathname === "/global/health") { + json(res, { healthy: true, version: "7.8.3" }); + return; + } + const dir = url.searchParams.get("directory")!; + const sessionId = dir === "/a" ? "ses_a" : "ses_b"; + if (url.pathname !== "/event") { + json(res, { id: sessionId, directory: dir }); + return; + } + res.writeHead(200, { "Content-Type": "text/event-stream" }); + streams.push({ res, sessionId }); + if (streams.length === 2) + for (const stream of streams) { + for (const id of ["ses_a", "ses_b"]) + stream.res.write( + `data: ${JSON.stringify({ type: "message.part.delta", properties: { sessionID: id, messageID: "same", partID: "same", field: "text", delta: id } })}\n\n`, + ); + } + }); + const [a, b] = await Promise.all( + ["a", "b"].map((id) => + run(KiloSessionClient.make({ instanceId: id, directory: `/${id}`, baseUrl })), + ), + ); + const received = await Promise.all( + [a!, b!].map((client, index) => + run( + client + .events({ + instanceId: index === 0 ? "a" : "b", + directory: index === 0 ? "/a" : "/b", + sessionId: index === 0 ? "ses_a" : "ses_b", + }) + .pipe(Stream.take(1), Stream.runCollect, Effect.scoped), + ), + ), + ); + expect( + received.map((events) => + events.map((e) => (e.type === "message.part.delta" ? e.properties.delta : "unexpected")), + ), + ).toEqual([["ses_a"], ["ses_b"]]); + }); + + it.each(["permission", "question"] as const)( + "answers an owned %s once and rejects stale answers", + async (kind) => { + let pending = true; + const replies: unknown[] = []; + const client = await withClient((req, res) => { + if (req.method === "GET") { + json( + res, + req.url!.startsWith(`/${kind}`) + ? pending + ? [{ id: "request", sessionID: ref.sessionId }] + : [] + : { id: ref.sessionId, directory }, + ); + return; + } + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", () => { + replies.push(JSON.parse(body)); + pending = false; + json(res, true); + }); + }); + const reply = () => + kind === "permission" + ? client.replyPermission(ref, "request", "once") + : client.replyQuestion(ref, "request", [["yes"]]); + await run(reply()); + expect((await run(reply().pipe(Effect.flip))).reason).toBe("wrong_owner"); + expect(replies).toEqual([kind === "permission" ? { reply: "once" } : { answers: [["yes"]] }]); + }, + ); +}); diff --git a/apps/server/src/provider/kilo/KiloSessionClient.ts b/apps/server/src/provider/kilo/KiloSessionClient.ts new file mode 100644 index 000000000000..473380409a6b --- /dev/null +++ b/apps/server/src/provider/kilo/KiloSessionClient.ts @@ -0,0 +1,409 @@ +import { createKiloClient, type Event, type KiloClient } from "@kilocode/sdk/v2"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +/** Persist with the T3 session. A native id alone is not an account or workspace identity. */ +const KiloSessionRef = Schema.Struct({ + instanceId: Schema.NonEmptyString, + directory: Schema.NonEmptyString, + sessionId: Schema.NonEmptyString, +}); +export type KiloSessionRef = typeof KiloSessionRef.Type; + +export class KiloSessionError extends Schema.TaggedError()("KiloSessionError", { + operation: Schema.String, + reason: Schema.Literals([ + "request_failed", + "admission_unknown", + "wrong_owner", + "unsupported_version", + "invalid_response", + ]), + cause: Schema.optional(Schema.Defect()), +}) { + override get message(): string { + return `Kilo ${this.operation} failed (${this.reason}).`; + } +} + +const Health = Schema.Struct({ healthy: Schema.Literal(true), version: Schema.Literal("7.8.3") }); +const decodeHealth = Schema.decodeUnknownEffect(Health); + +const SessionOwner = Schema.Struct({ id: Schema.NonEmptyString, directory: Schema.String }); +const PendingOwners = Schema.Array( + Schema.Struct({ id: Schema.NonEmptyString, sessionID: Schema.NonEmptyString }), +); +const EventEnvelope = Schema.Struct({ + type: Schema.NonEmptyString, + properties: Schema.Record(Schema.String, Schema.Unknown), +}); + +const isKiloSessionError = Schema.is(KiloSessionError); +const isSyncEnvelope = Schema.is( + Schema.Struct({ + type: Schema.Literal("sync"), + syncEvent: Schema.Record(Schema.String, Schema.Unknown), + }), +); +const isSessionOwner = Schema.is(SessionOwner); +const isPendingOwners = Schema.is(PendingOwners); +const isEventEnvelope = Schema.is(EventEnvelope); +const isRecord = Schema.is(Schema.Record(Schema.String, Schema.Unknown)); + +function sessionIdOf(event: Event): Effect.Effect { + // The server also sends replication envelopes omitted from its generated Event union. + // Ordinary session/message events follow these and remain the source for this stream. + if (isSyncEnvelope(event)) return Effect.succeed(undefined); + if (!isEventEnvelope(event)) { + return Effect.fail( + new KiloSessionError({ operation: "event.subscribe", reason: "invalid_response" }), + ); + } + const properties = event.properties; + let value: unknown; + if ("sessionID" in properties) value = properties.sessionID; + else if (event.type === "message.part.updated") { + value = isRecord(properties.part) ? properties.part.sessionID : undefined; + } else if (event.type === "message.updated") { + value = isRecord(properties.info) ? properties.info.sessionID : undefined; + } else if ( + event.type === "session.created" || + event.type === "session.updated" || + event.type === "session.deleted" + ) { + value = isRecord(properties.info) ? properties.info.id : undefined; + } else if (event.type === "session.error") { + return Effect.fail( + new KiloSessionError({ operation: "event.subscribe", reason: "request_failed" }), + ); + } else { + // Directory streams include global config, PTY and health events. Their `info` + // fields are not session records and must not terminate unrelated sessions. + return Effect.succeed(undefined); + } + if (typeof value !== "string" || value.length === 0) { + return Effect.fail( + new KiloSessionError({ operation: "event.subscribe", reason: "invalid_response" }), + ); + } + return Effect.succeed(value); +} + +/** + * One local Kilo provider instance and directory, using Kilo's SDK and auth/header conventions. + * No ambient credentials, cloud routes, automatic retries of mutations, or global config writes. + * The caller owns the server process and supplies its scoped loopback URL. + * Account changes must invalidate the instance identity before restoring saved refs. + */ +export const make = Effect.fn("KiloSessionClient.make")(function* (input: { + readonly instanceId: string; + readonly directory: string; + readonly baseUrl: string; + readonly serverPassword?: string; + readonly serverUsername?: string; +}) { + const client = createKiloClient({ + baseUrl: input.baseUrl, + directory: input.directory, + throwOnError: true, + // Mutation redirects must never forward an authorization header or repeat a prompt elsewhere. + redirect: "error", + ...(input.serverPassword === undefined + ? {} + : { + headers: { + Authorization: `Basic ${Buffer.from(`${input.serverUsername ?? "kilo"}:${input.serverPassword}`).toString("base64")}`, + }, + }), + }); + + const request = (operation: string, run: (signal: AbortSignal) => Promise<{ data?: A }>) => + Effect.tryPromise({ + try: run, + catch: (cause) => new KiloSessionError({ operation, reason: "request_failed", cause }), + }).pipe( + Effect.timeout("10 seconds"), + Effect.catchTag( + "TimeoutError", + (cause) => new KiloSessionError({ operation, reason: "request_failed", cause }), + ), + Effect.flatMap((response) => + response.data === undefined + ? Effect.fail(new KiloSessionError({ operation, reason: "invalid_response" })) + : Effect.succeed(response.data), + ), + ); + + const acknowledge = (operation: string) => (accepted: unknown) => + accepted === true + ? Effect.void + : Effect.fail(new KiloSessionError({ operation, reason: "invalid_response" })); + + const checkOwner = (ref: KiloSessionRef, operation: string) => + ref.instanceId !== input.instanceId || ref.directory !== input.directory + ? Effect.fail(new KiloSessionError({ operation, reason: "wrong_owner" })) + : Effect.void; + + // Kilo's GET /session/{id} can resolve an id from another directory. The SDK directory + // header is routing context, not authorization. Check the returned owner before every mutation. + const read = (ref: KiloSessionRef) => + checkOwner(ref, "session.get").pipe( + Effect.andThen( + request("session.get", (signal) => + client.session.get({ sessionID: ref.sessionId }, { signal }), + ), + ), + Effect.flatMap((session) => + isSessionOwner(session) && + session.id === ref.sessionId && + session.directory === input.directory + ? Effect.succeed(session) + : Effect.fail(new KiloSessionError({ operation: "session.get", reason: "wrong_owner" })), + ), + ); + + const owned = ( + ref: KiloSessionRef, + operation: string, + run: (signal: AbortSignal) => Promise<{ data?: A }>, + ) => read(ref).pipe(Effect.andThen(request(operation, run))); + + const reference = (session: { id: string; directory: string }): KiloSessionRef => ({ + instanceId: input.instanceId, + directory: session.directory, + sessionId: session.id, + }); + + const health = yield* request("global.health", (signal) => client.global.health({ signal })).pipe( + Effect.flatMap((value) => + decodeHealth(value).pipe( + Effect.mapError( + (cause) => + new KiloSessionError({ + operation: "global.health", + reason: "unsupported_version", + cause, + }), + ), + ), + ), + ); + + return { + version: health.version, + models: () => request("provider.list", (signal) => client.provider.list(undefined, { signal })), + agents: () => request("app.agents", (signal) => client.app.agents(undefined, { signal })), + create: ( + permission: NonNullable[0]>["permission"], + ) => + request("session.create", (signal) => + client.session.create(permission === undefined ? {} : { permission }, { signal }), + ).pipe( + Effect.flatMap((session) => + isSessionOwner(session) && session.directory === input.directory + ? Effect.succeed(reference(session)) + : Effect.fail( + new KiloSessionError({ operation: "session.create", reason: "wrong_owner" }), + ), + ), + ), + read, + history: (ref: KiloSessionRef) => + owned(ref, "session.messages", (signal) => + client.session.messages({ sessionID: ref.sessionId }, { signal }), + ), + fork: (ref: KiloSessionRef, messageID?: string) => + owned(ref, "session.fork", (signal) => + client.session.fork( + { sessionID: ref.sessionId, ...(messageID === undefined ? {} : { messageID }) }, + { signal }, + ), + ).pipe( + Effect.flatMap((session) => + isSessionOwner(session) && session.directory === input.directory + ? Effect.succeed(reference(session)) + : Effect.fail( + new KiloSessionError({ operation: "session.fork", reason: "wrong_owner" }), + ), + ), + ), + // Native revert can modify files. The orchestration adapter must coordinate it with + // T3 checkpoints and exclude concurrent writers before exposing this operation. + revert: (ref: KiloSessionRef, messageID: string) => + owned(ref, "session.revert", (signal) => + client.session.revert({ sessionID: ref.sessionId, messageID }, { signal }), + ), + prompt: ( + ref: KiloSessionRef, + prompt: Omit< + Parameters[0], + "sessionID" | "directory" | "workspace" + >, + ) => + read(ref).pipe( + Effect.andThen( + Effect.tryPromise({ + try: (signal) => + client.session.promptAsync( + { + ...prompt, + sessionID: ref.sessionId, + directory: input.directory, + }, + { signal }, + ), + catch: (cause) => + new KiloSessionError({ + operation: "session.promptAsync", + reason: "admission_unknown", + cause, + }), + }).pipe( + Effect.timeout("10 seconds"), + Effect.catchTag( + "TimeoutError", + (cause) => + new KiloSessionError({ + operation: "session.promptAsync", + reason: "admission_unknown", + cause, + }), + ), + ), + ), + Effect.flatMap((response) => + response.response?.status === 204 + ? Effect.void + : Effect.fail( + new KiloSessionError({ + operation: "session.promptAsync", + reason: "invalid_response", + }), + ), + ), + ), + abort: (ref: KiloSessionRef) => + owned(ref, "session.abort", (signal) => + client.session.abort({ sessionID: ref.sessionId }, { signal }), + ).pipe( + Effect.flatMap((accepted) => + accepted === true + ? Effect.void + : Effect.fail( + new KiloSessionError({ operation: "session.abort", reason: "invalid_response" }), + ), + ), + ), + replyPermission: ( + ref: KiloSessionRef, + requestID: string, + reply: "once" | "always" | "reject", + ) => + owned(ref, "permission.list", (signal) => client.permission.list(undefined, { signal })).pipe( + Effect.flatMap((pending) => + isPendingOwners(pending) && + pending.some((p) => p.id === requestID && p.sessionID === ref.sessionId) + ? request("permission.reply", (signal) => + client.permission.reply({ requestID, reply }, { signal }), + ).pipe(Effect.flatMap(acknowledge("permission.reply"))) + : Effect.fail( + new KiloSessionError({ operation: "permission.reply", reason: "wrong_owner" }), + ), + ), + ), + replyQuestion: (ref: KiloSessionRef, requestID: string, answers: string[][]) => + owned(ref, "question.list", (signal) => client.question.list(undefined, { signal })).pipe( + Effect.flatMap((pending) => + isPendingOwners(pending) && + pending.some((p) => p.id === requestID && p.sessionID === ref.sessionId) + ? request("question.reply", (signal) => + client.question.reply({ requestID, answers }, { signal }), + ).pipe(Effect.flatMap(acknowledge("question.reply"))) + : Effect.fail( + new KiloSessionError({ operation: "question.reply", reason: "wrong_owner" }), + ), + ), + ), + events: (ref: KiloSessionRef) => + Stream.unwrap( + read(ref).pipe( + Effect.andThen( + Effect.gen(function* () { + const controller = new AbortController(); + yield* Effect.addFinalizer(() => Effect.sync(() => controller.abort())); + let streamFailure: unknown; + const subscription = yield* Effect.tryPromise({ + try: () => + client.event.subscribe(undefined, { + signal: controller.signal, + sseMaxRetryAttempts: 0, + onSseError: (cause) => { + streamFailure = cause; + }, + }), + catch: (cause) => + new KiloSessionError({ + operation: "event.subscribe", + reason: "request_failed", + cause, + }), + }); + const interruptible: AsyncIterable = { + [Symbol.asyncIterator]() { + const iterator = subscription.stream[Symbol.asyncIterator](); + return { + next: () => iterator.next(), + return: async () => { + // Abort a pending reader.read before awaiting generator cleanup. + // A scope finalizer alone runs after fromAsyncIterable's finalizer. + controller.abort(); + return iterator.return + ? iterator.return() + : { done: true as const, value: undefined }; + }, + }; + }, + }; + return Stream.fromAsyncIterable( + interruptible, + (cause) => + new KiloSessionError({ + operation: "event.subscribe", + reason: "request_failed", + cause, + }), + ).pipe( + Stream.timeout("45 seconds"), + Stream.mapError((cause) => + isKiloSessionError(cause) + ? cause + : new KiloSessionError({ + operation: "event.subscribe", + reason: "request_failed", + cause, + }), + ), + Stream.filterEffect((event) => + sessionIdOf(event).pipe(Effect.map((sessionId) => sessionId === ref.sessionId)), + ), + Stream.concat( + Stream.unwrap( + Effect.sync(() => + Stream.fail( + new KiloSessionError({ + operation: "event.subscribe", + reason: "request_failed", + cause: streamFailure, + }), + ), + ), + ), + ), + ); + }), + ), + ), + ), + }; +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 092af1eaeeb7..668bef338dfd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -526,6 +526,9 @@ importers: '@ff-labs/fff-node': specifier: 0.9.4 version: 0.9.4(patch_hash=c4e3cc2420ceb9dc650f9d189e9c24baf7e83f342998bacda5f459a4ce7927a8) + '@kilocode/sdk': + specifier: 7.8.3 + version: 7.8.3 '@napi-rs/keyring': specifier: ^1.3.0 version: 1.3.0 @@ -3323,6 +3326,9 @@ packages: resolution: {integrity: sha512-hloP58zRVCRSpgDxmqCWJNlizAlUgJFqG2ypq79DCvyv9tHjRYMDOcPFjzfl/A1/YxDvRCZz8wvZvmapQnKwFQ==} engines: {node: '>=12'} + '@kilocode/sdk@7.8.3': + resolution: {integrity: sha512-Wk0XaXX+NNezo03k5FEXeWaBF+wE1wtzIFfztkvkXlJ0Grfo8VRQE2MrM7WX1wuLlrAacGrFIFLISYRIpqOmpQ==} + '@legendapp/list@3.3.5': resolution: {integrity: sha512-XTsLYtpg41SVb5uLBYA+YcDSA3w0tgoPq/W8ZggQ2tx+3lrC/rf+ehTP9KYHea9oFaZIuePAgzACs5/auVMJlQ==} peerDependencies: @@ -13595,6 +13601,10 @@ snapshots: dependencies: jsbi: 4.3.2 + '@kilocode/sdk@7.8.3': + dependencies: + cross-spawn: 7.0.6 + '@legendapp/list@3.3.5(patch_hash=680cc6a5c5b4a4032e467e7b3fde22f89a84c0ee2e6eac6fda737d6277cc0806)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': dependencies: react: 19.2.6 From dfa68127e1a4a992b4184babe0e193432465bdd2 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 09:18:52 +0000 Subject: [PATCH 02/44] ci: verify Kilo sessions on the fork Linux runner --- .github/workflows/kilo-provider.yml | 62 +++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/kilo-provider.yml diff --git a/.github/workflows/kilo-provider.yml b/.github/workflows/kilo-provider.yml new file mode 100644 index 000000000000..f7dee0da7706 --- /dev/null +++ b/.github/workflows/kilo-provider.yml @@ -0,0 +1,62 @@ +name: Kilo provider checks + +on: + pull_request: + paths: + - .github/workflows/kilo-provider.yml + - apps/server/src/provider/kilo/** + - apps/server/package.json + - pnpm-lock.yaml + +permissions: + contents: read + +concurrency: + group: kilo-provider-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + local-sdk: + # The upstream suite uses Blacksmith runners unavailable to this fork. + # Reassess this workflow before proposing the provider upstream. + if: github.repository == 'Githubguy132010/t3code' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + with: + # Verify the exact PR head rather than an implicit merge commit. + ref: ${{ github.event.pull_request.head.sha }} + + - uses: voidzero-dev/setup-vp@v1 + with: + node-version-file: package.json + cache: true + run-install: true + + - name: Install the pinned local Kilo CLI + run: npm install --prefix "$RUNNER_TEMP/kilo-probe" --ignore-scripts @kilocode/cli@7.8.3 + + - name: Lint the Kilo client + run: vp lint apps/server/src/provider/kilo --deny-warnings + + - name: Check server types + run: vp exec tsc --noEmit -p apps/server + + - name: Check server dependencies and exports + run: vp exec knip --workspace apps/server --include files,dependencies,exports --preprocessor ./scripts/knip-schemas.ts --no-config-hints + + - name: Test native Kilo sessions and OpenCode regressions + env: + KILO_BIN: ${{ runner.temp }}/kilo-probe/node_modules/.bin/kilo + KILO_TEST_ROOT: ${{ runner.temp }} + run: >- + vp test run + apps/server/src/provider/kilo + apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.test.ts + apps/server/src/provider/opencodeRuntime.environment.test.ts + apps/server/src/provider/opencodeRuntime.permissions.test.ts + apps/server/src/provider/Drivers/OpenCodeDriver.test.ts + + - name: Build the server bundle + run: vp run --filter t3 build:bundle From b0c086d990889e94a8d42977dde7cae15baf848f Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 11:47:11 +0000 Subject: [PATCH 03/44] feat(provider): integrate isolated local Kilo sessions --- .github/workflows/kilo-provider.yml | 79 +- apps/desktop/scripts/kilo-ui-evidence.mjs | 234 +++ apps/mobile/src/components/ProviderIcon.tsx | 10 + .../SettingsProviderAccountsRouteScreen.tsx | 30 +- apps/server/scripts/kilo-stream-benchmark.mjs | 135 ++ .../Adapters/KiloAdapterV2.live.test.ts | 893 +++++++++ .../Adapters/KiloAdapterV2.ts | 1677 +++++++++++++++++ .../server/src/provider/Drivers/KiloDriver.ts | 270 +++ apps/server/src/provider/builtInDrivers.ts | 5 +- .../src/provider/kilo/KiloCloudClient.test.ts | 149 ++ .../src/provider/kilo/KiloCloudClient.ts | 214 +++ .../provider/kilo/KiloRuntime.live.test.ts | 114 ++ apps/server/src/provider/kilo/KiloRuntime.ts | 217 +++ .../provider/kilo/KiloSessionClient.test.ts | 66 + .../src/provider/kilo/KiloSessionClient.ts | 202 +- .../src/textGeneration/KiloTextGeneration.ts | 68 + apps/web/src/components/Icons.tsx | 9 + .../components/chat/ProviderInstanceIcon.tsx | 2 + .../components/settings/providerDriverMeta.ts | 17 + docs/user/providers-kilo.md | 37 + knip.jsonc | 2 + packages/client-runtime/package.json | 4 + packages/client-runtime/src/kiloIcon.ts | 8 + packages/contracts/src/orchestrationV2.ts | 24 + packages/contracts/src/settings.ts | 35 + third-party-licenses.config.json | 24 + 26 files changed, 4481 insertions(+), 44 deletions(-) create mode 100644 apps/desktop/scripts/kilo-ui-evidence.mjs create mode 100644 apps/server/scripts/kilo-stream-benchmark.mjs create mode 100644 apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts create mode 100644 apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts create mode 100644 apps/server/src/provider/Drivers/KiloDriver.ts create mode 100644 apps/server/src/provider/kilo/KiloCloudClient.test.ts create mode 100644 apps/server/src/provider/kilo/KiloCloudClient.ts create mode 100644 apps/server/src/provider/kilo/KiloRuntime.live.test.ts create mode 100644 apps/server/src/provider/kilo/KiloRuntime.ts create mode 100644 apps/server/src/textGeneration/KiloTextGeneration.ts create mode 100644 docs/user/providers-kilo.md create mode 100644 packages/client-runtime/src/kiloIcon.ts diff --git a/.github/workflows/kilo-provider.yml b/.github/workflows/kilo-provider.yml index f7dee0da7706..13b3a305df4b 100644 --- a/.github/workflows/kilo-provider.yml +++ b/.github/workflows/kilo-provider.yml @@ -5,6 +5,18 @@ on: paths: - .github/workflows/kilo-provider.yml - apps/server/src/provider/kilo/** + - apps/server/src/provider/Drivers/KiloDriver.ts + - apps/server/src/provider/builtInDrivers.ts + - apps/server/src/orchestration-v2/Adapters/KiloAdapterV2* + - apps/server/src/textGeneration/KiloTextGeneration.ts + - apps/server/scripts/kilo-stream-benchmark.mjs + - apps/desktop/scripts/kilo-ui-evidence.mjs + - apps/web/src/components/** + - apps/mobile/src/** + - packages/contracts/src/** + - packages/client-runtime/** + - third-party-licenses.config.json + - knip.jsonc - apps/server/package.json - pnpm-lock.yaml @@ -16,12 +28,12 @@ concurrency: cancel-in-progress: true jobs: - local-sdk: + provider-integration: # The upstream suite uses Blacksmith runners unavailable to this fork. # Reassess this workflow before proposing the provider upstream. if: github.repository == 'Githubguy132010/t3code' runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 25 steps: - uses: actions/checkout@v6 with: @@ -37,14 +49,38 @@ jobs: - name: Install the pinned local Kilo CLI run: npm install --prefix "$RUNNER_TEMP/kilo-probe" --ignore-scripts @kilocode/cli@7.8.3 - - name: Lint the Kilo client - run: vp lint apps/server/src/provider/kilo --deny-warnings + - name: Lint changed provider and client code + run: >- + vp lint apps/server/src/provider/kilo + apps/server/src/provider/Drivers/KiloDriver.ts + apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts + apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts + apps/server/src/textGeneration/KiloTextGeneration.ts + apps/server/scripts/kilo-stream-benchmark.mjs + apps/desktop/scripts/kilo-ui-evidence.mjs + apps/web/src/components/Icons.tsx + apps/web/src/components/chat/ProviderInstanceIcon.tsx + apps/web/src/components/settings/providerDriverMeta.ts + packages/client-runtime/src/kiloIcon.ts + packages/contracts/src/settings.ts + packages/contracts/src/orchestrationV2.ts --deny-warnings + + - name: Lint mobile changes + run: vp lint apps/mobile/src/components/ProviderIcon.tsx apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx - - name: Check server types - run: vp exec tsc --noEmit -p apps/server + - name: Check changed workspace types + run: | + vp exec tsc --noEmit -p apps/server + vp exec tsc --noEmit -p apps/web + vp exec tsc --noEmit -p apps/mobile + vp exec tsc --noEmit -p packages/contracts + vp exec tsc --noEmit -p packages/client-runtime - name: Check server dependencies and exports - run: vp exec knip --workspace apps/server --include files,dependencies,exports --preprocessor ./scripts/knip-schemas.ts --no-config-hints + run: vp exec knip --workspace apps/server --workspace apps/web --workspace packages/contracts --workspace packages/client-runtime --include files,dependencies,exports --preprocessor ./scripts/knip-schemas.ts --no-config-hints + + - name: Check mobile files and dependencies + run: vp exec knip --workspace apps/mobile --include files,dependencies --no-config-hints - name: Test native Kilo sessions and OpenCode regressions env: @@ -53,6 +89,7 @@ jobs: run: >- vp test run apps/server/src/provider/kilo + apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.test.ts apps/server/src/provider/opencodeRuntime.environment.test.ts apps/server/src/provider/opencodeRuntime.permissions.test.ts @@ -60,3 +97,31 @@ jobs: - name: Build the server bundle run: vp run --filter t3 build:bundle + + - name: Build web and desktop bundles + run: | + vp run --filter @t3tools/web build + vp run --filter @t3tools/desktop build + + - name: Export the Android JavaScript bundle + working-directory: apps/mobile + env: + APP_VARIANT: development + run: vp exec expo export --platform android --output-dir "$RUNNER_TEMP/kilo-android" + + - name: Install Chromium for the authorized UI check + run: node apps/desktop/node_modules/playwright-core/cli.js install --with-deps chromium + + - name: Exercise the real UI with local inference + env: + KILO_BIN: ${{ runner.temp }}/kilo-probe/node_modules/.bin/kilo + KILO_EVIDENCE_DIR: ${{ runner.temp }}/kilo-evidence + run: node apps/desktop/scripts/kilo-ui-evidence.mjs + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: kilo-ui-${{ github.event.pull_request.head.sha }} + path: ${{ runner.temp }}/kilo-evidence + if-no-files-found: warn + retention-days: 14 diff --git a/apps/desktop/scripts/kilo-ui-evidence.mjs b/apps/desktop/scripts/kilo-ui-evidence.mjs new file mode 100644 index 000000000000..8792d6c4312f --- /dev/null +++ b/apps/desktop/scripts/kilo-ui-evidence.mjs @@ -0,0 +1,234 @@ +// Real web UI + Orchestrator + pinned Kilo CLI. All inference stays on this loopback fixture. +// KILO_BIN=/path/to/kilo KILO_EVIDENCE_DIR=/tmp/evidence node apps/desktop/scripts/kilo-ui-evidence.mjs +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeOS from "node:os"; +import * as NodeHttp from "node:http"; +import * as NodeEvents from "node:events"; +import * as NodeChildProcess from "node:child_process"; +import * as NodeUtil from "node:util"; +import { chromium } from "playwright-core"; + +if (!process.env.KILO_BIN) throw new Error("KILO_BIN must point to the pinned local CLI"); +const root = NodePath.resolve(import.meta.dirname, "../../.."); +const temporary = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-kilo-ui-")); +const evidence = process.env.KILO_EVIDENCE_DIR ?? NodePath.join(temporary, "evidence"); +const state = NodePath.join(temporary, "state"); +const workspace = NodePath.join(temporary, "kilo-ui-workspace"); +await Promise.all([ + NodeFSP.mkdir(evidence, { recursive: true }), + NodeFSP.mkdir(workspace), + NodeFSP.mkdir(NodePath.join(state, "userdata"), { recursive: true }), +]); +const execFile = NodeUtil.promisify(NodeChildProcess.execFile); +await NodeFSP.writeFile(NodePath.join(workspace, "README.md"), "Local Kilo UI fixture\n"); +for (const args of [ + ["init"], + ["add", "."], + [ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-m", + "fixture", + ], +]) + await execFile("git", args, { cwd: workspace }); +let requests = 0; +const answer = + "Kilo local integration succeeded. This response came from the isolated local fixture."; +const model = NodeHttp.createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", () => { + requests++; + const data = JSON.parse(body); + const prompt = JSON.stringify(data.messages ?? []); + const text = + prompt.includes("title") && prompt.includes("JSON") + ? '{"title":"Kilo local verification"}' + : answer; + res.writeHead(200, { "Content-Type": data.stream ? "text/event-stream" : "application/json" }); + if (!data.stream) { + res.end( + JSON.stringify({ + id: "local", + object: "chat.completion", + created: 0, + model: "test", + choices: [ + { index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }, + ], + }), + ); + return; + } + for (const chunk of text.match(/.{1,16}/g)) + res.write( + `data: ${JSON.stringify({ + id: "local", + object: "chat.completion.chunk", + created: 0, + model: "test", + choices: [{ index: 0, delta: { content: chunk }, finish_reason: null }], + })}\n\n`, + ); + res.end( + `data: ${JSON.stringify({ + id: "local", + object: "chat.completion.chunk", + created: 0, + model: "test", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 12, completion_tokens: 10, total_tokens: 22 }, + })}\n\ndata: [DONE]\n\n`, + ); + }); +}); +model.listen(0, "127.0.0.1"); +await NodeEvents.once(model, "listening"); +const config = { + model: "fixture/test", + small_model: "fixture/test", + plugin: [], + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + name: "Local fixture", + options: { baseURL: `http://127.0.0.1:${model.address().port}/v1` }, + models: { test: { name: "Local fixture", limit: { context: 10000, output: 1000 } } }, + }, + }, +}; +await NodeFSP.writeFile( + NodePath.join(state, "userdata/settings.json"), + JSON.stringify({ + providers: Object.fromEntries( + ["codex", "claudeAgent", "cursor", "grok", "opencode", "antigravity", "pi"].map((name) => [ + name, + { enabled: false }, + ]), + ), + providerInstances: { + kilo: { + driver: "kilo", + displayName: "Kilo", + enabled: true, + config: { binaryPath: process.env.KILO_BIN, accountId: "ui-fixture" }, + environment: [ + { name: "KILO_CONFIG_CONTENT", value: JSON.stringify(config) }, + ...[ + "KILO_DISABLE_MODELS_FETCH", + "KILO_DISABLE_DEFAULT_PLUGINS", + "KILO_DISABLE_EXTERNAL_SKILLS", + "KILO_DISABLE_PROJECT_CONFIG", + ].map((name) => ({ name, value: "1" })), + ], + }, + }, + textGenerationModelSelection: { instanceId: "kilo", model: "fixture/test", options: [] }, + }), +); +const child = NodeChildProcess.spawn("vp", ["run", "dev", "--home-dir", state], { + cwd: root, + detached: true, + stdio: ["ignore", "pipe", "pipe"], +}); +let browser; +let page; +try { + const pair = await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error("Isolated T3 did not become ready")), 120000); + let output = ""; + const read = (chunk) => { + output = (output + chunk).slice(-50000); + // Startup output may contain ANSI color escapes immediately after the URL. + // oxlint-disable-next-line no-control-regex + const match = /pairingUrl:\s*(http[^\s\x1b]+)/.exec(output); + if (match) { + clearTimeout(timeout); + resolve(match[1]); + } + }; + child.stdout.on("data", read); + child.stderr.on("data", read); + child.once("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.once("exit", (code) => { + clearTimeout(timeout); + reject(new Error(`T3 exited with ${code}`)); + }); + }); + browser = await chromium.launch({ + headless: true, + ...(process.env.CHROMIUM_PATH ? { executablePath: process.env.CHROMIUM_PATH } : {}), + args: ["--no-sandbox"], + }); + const context = await browser.newContext({ + viewport: { width: 1440, height: 1000 }, + recordVideo: { dir: evidence, size: { width: 1440, height: 1000 } }, + }); + page = await context.newPage(); + page.setDefaultTimeout(45000); + await page.goto(pair); + await page.getByRole("button", { name: "Add project", exact: true }).click(); + await page.getByText("Local folder", { exact: true }).click(); + await page.getByPlaceholder("Enter path (e.g. ~/projects/my-app)").fill(workspace); + await page.getByPlaceholder("Enter path (e.g. ~/projects/my-app)").press("Enter"); + await page.locator("[data-chat-provider-model-picker-label]").click(); + await page.getByPlaceholder("Search models...").fill("Local fixture"); + await page.getByText("Local fixture", { exact: true }).last().click(); + await page.getByRole("button", { name: "Local fixture", exact: true }).waitFor(); + await page.locator("[contenteditable=true]").fill("Kilo local integration: say hello."); + await page.screenshot({ path: NodePath.join(evidence, "before-send.png") }); + await page.getByRole("button", { name: "Submit message", exact: true }).click(); + await page.getByText(answer, { exact: true }).waitFor({ timeout: 60000 }); + await page.screenshot({ path: NodePath.join(evidence, "streamed-answer.png") }); + await page.getByRole("button", { name: "Submit message", exact: true }).waitFor(); + await page.screenshot({ path: NodePath.join(evidence, "completed-answer.png") }); + console.log("Local native answer rendered; opening provider settings."); + await page.getByRole("button", { name: "Settings", exact: true }).click(); + await page.waitForURL("**/settings/general*"); + await page.getByText("Restore device defaults", { exact: true }).waitFor(); + await page.getByRole("button", { name: "Providers", exact: true }).click(); + await page.waitForURL("**/settings/providers*"); + await page.getByRole("button", { name: "Add provider", exact: true }).waitFor(); + await page.screenshot({ path: NodePath.join(evidence, "provider-settings.png") }); + await context.close(); + if (!requests) throw new Error("The real CLI did not contact the local inference fixture"); + await NodeFSP.writeFile( + NodePath.join(evidence, "verification.json"), + JSON.stringify( + { + commit: (await execFile("git", ["rev-parse", "HEAD"], { cwd: root })).stdout.trim(), + inferenceRequests: requests, + inference: "loopback fixture only", + client: "Chromium web", + }, + null, + 2, + ), + ); + console.log("Kilo UI verification passed; screenshots and video saved."); +} catch (error) { + await page?.screenshot({ path: NodePath.join(evidence, "failure.png") }).catch(() => {}); + console.error(String(error).replace(/https?:\/\/[^\s)]+/g, "[local URL]")); + process.exitCode = 1; +} finally { + await browser?.close(); + // This is the process group captured at spawn, never a PID discovered by matching. + try { + process.kill(-child.pid, "SIGTERM"); + } catch { + /* already exited */ + } + model.closeAllConnections(); + await new Promise((resolve) => model.close(resolve)); +} diff --git a/apps/mobile/src/components/ProviderIcon.tsx b/apps/mobile/src/components/ProviderIcon.tsx index 964c60ebcbe2..bed044599bfc 100644 --- a/apps/mobile/src/components/ProviderIcon.tsx +++ b/apps/mobile/src/components/ProviderIcon.tsx @@ -1,3 +1,4 @@ +import { kiloIconPaths } from "@t3tools/client-runtime/kilo-icon"; import { Image } from "expo-image"; import { Path, Svg } from "react-native-svg"; import { View } from "react-native"; @@ -133,6 +134,15 @@ export function ProviderIcon(props: ProviderIconProps) { ); } + if (props.provider === "kilo") { + return ( + + {kiloIconPaths.map((path) => ( + + ))} + + ); + } if (props.provider === "opencode") { return ( diff --git a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx index e0f3e2ecc69e..5fda519517ae 100644 --- a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx @@ -8,6 +8,7 @@ import { useRef, useState } from "react"; import { Alert, Linking, Pressable, ScrollView, TextInput, View } from "react-native"; import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { ProviderIcon } from "../../components/ProviderIcon"; import { AppText as Text } from "../../components/AppText"; import { ScreenScrollView } from "../../components/ScreenScrollView"; import { useEnvironmentQuery } from "../../state/query"; @@ -43,18 +44,33 @@ export function SettingsProviderAccountsRouteScreen() { {environment.serverConfig.providers .filter( (provider) => + provider.driver === "kilo" || provider.setup?.canAuthenticate || (provider.driver === "acpRegistry" && provider.installed), ) - .map((provider) => ( - - ))} + .map((provider) => + provider.driver === "kilo" ? ( + + + + {provider.displayName} + + + {provider.message ?? + "Kilo runs on this environment. Manage its isolated account profile in web or desktop Settings."} + + + ) : ( + + ), + )} {!environment.serverConfig.providers.some( (provider) => + provider.driver === "kilo" || provider.setup?.canAuthenticate || (provider.driver === "acpRegistry" && provider.installed), ) ? ( diff --git a/apps/server/scripts/kilo-stream-benchmark.mjs b/apps/server/scripts/kilo-stream-benchmark.mjs new file mode 100644 index 000000000000..ef2202276bd5 --- /dev/null +++ b/apps/server/scripts/kilo-stream-benchmark.mjs @@ -0,0 +1,135 @@ +// node --expose-gc apps/server/scripts/kilo-stream-benchmark.mjs /absolute/baseline/KiloSessionClient.ts +// Extract the unchanged client with git show, beside a node_modules link to apps/server/node_modules. +// Baseline and current perform the same ownership read, filtering, validation and consumption. +// The raw SDK is diagnostic only: it does less work and is not a regression baseline. +import * as NodeHttp from "node:http"; +import * as NodeEvents from "node:events"; +import * as NodeURL from "node:url"; +import * as NodeOS from "node:os"; +import * as NodeAssert from "node:assert/strict"; +import { createKiloClient } from "@kilocode/sdk/v2"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; +import { make } from "../src/provider/kilo/KiloSessionClient.ts"; + +if (!process.argv[2] || !global.gc) throw new Error("Pass a baseline path and --expose-gc"); +const baseline = await import(NodeURL.pathToFileURL(process.argv[2]).href); +const ref = { instanceId: "bench", directory: "/bench", sessionId: "ses_bench" }; +let workload = { count: 10000, intervalMs: 0 }; +const server = NodeHttp.createServer((req, res) => { + if (!req.url.startsWith("/event")) { + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify( + req.url.startsWith("/global/health") + ? { healthy: true, version: "7.8.3" } + : { id: ref.sessionId, directory: ref.directory }, + ), + ); + return; + } + res.writeHead(200, { "content-type": "text/event-stream" }); + const event = (i) => + `data: ${JSON.stringify({ + type: "message.part.delta", + properties: { + sessionID: i % 2 ? "ses_foreign" : ref.sessionId, + messageID: "msg", + partID: "part", + field: "text", + delta: JSON.stringify({ i, sentAt: performance.now() }), + }, + })}\n\n`; + if (!workload.intervalMs) { + res.end(Array.from({ length: workload.count }, (_, i) => event(i)).join("")); + return; + } + let index = 0; + const timer = setInterval(() => { + res.write(event(index++)); + if (index === workload.count) { + clearInterval(timer); + res.end(); + } + }, workload.intervalMs); + res.on("close", () => clearInterval(timer)); +}); +server.listen(0, "127.0.0.1"); +await NodeEvents.once(server, "listening"); +const baseUrl = `http://127.0.0.1:${server.address().port}`; +const clients = { + baseline: await Effect.runPromise(baseline.make({ ...ref, baseUrl })), + current: await Effect.runPromise(make({ ...ref, baseUrl })), +}; +const sdk = createKiloClient({ baseUrl, directory: ref.directory, throwOnError: true }); +const results = []; +try { + for (const scenario of ["burst", "paced"]) { + workload = + scenario === "burst" ? { count: 10000, intervalMs: 0 } : { count: 200, intervalMs: 5 }; + for (let round = 0; round < 8; round++) { + const order = round % 2 ? ["current", "baseline", "sdk"] : ["sdk", "baseline", "current"]; + for (const kind of order) { + global.gc(); + const heap = process.memoryUsage().heapUsed; + const cpu = process.cpuUsage(); + const began = performance.now(); + const indices = []; + const latency = []; + const consume = (event) => { + const delta = JSON.parse(event.properties.delta); + indices.push(delta.i); + latency.push(performance.now() - delta.sentAt); + }; + if (kind === "sdk") { + const subscription = await sdk.event.subscribe(undefined, { sseMaxRetryAttempts: 0 }); + for await (const event of subscription.stream) + if (event.properties.sessionID === ref.sessionId) consume(event); + } else { + const failure = await Effect.runPromise( + clients[kind].events(ref).pipe( + Stream.runForEach((event) => Effect.sync(() => consume(event))), + Effect.scoped, + Effect.flip, + ), + ); + NodeAssert.equal(failure.reason, "request_failed"); // EOF is not task completion. + } + const wallMs = performance.now() - began; + const used = process.cpuUsage(cpu); + NodeAssert.deepEqual( + indices, + Array.from({ length: workload.count / 2 }, (_, i) => i * 2), + ); + latency.sort((a, b) => a - b); + results.push({ + scenario, + round, + kind, + retained: indices.length, + wallMs, + cpuMs: (used.user + used.system) / 1000, + heapDelta: process.memoryUsage().heapUsed - heap, + latencyP50Ms: latency[Math.floor(latency.length * 0.5)], + latencyP95Ms: latency[Math.floor(latency.length * 0.95)], + }); + } + } + } + console.log( + JSON.stringify( + { + node: process.version, + cpu: NodeOS.cpus()[0]?.model, + baselinePath: process.argv[2], + buffering: "HTTP/SSE to serial consumer; no added queue", + results, + }, + null, + 2, + ), + ); +} finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); +} diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts new file mode 100644 index 000000000000..d257b0b23e03 --- /dev/null +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts @@ -0,0 +1,893 @@ +// @effect-diagnostics nodeBuiltinImport:off - local inference wire fixture for the real Kilo CLI. +import * as NodeHttp from "node:http"; +import * as NodeEvents from "node:events"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { + MessageId, + ChatAttachmentId, + CommandId, + CheckpointId, + NodeId, + ProjectId, + ProviderInstanceId, + ProviderSessionId, + RunAttemptId, + RunId, + ThreadId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Stream from "effect/Stream"; +import * as Schema from "effect/Schema"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import * as EffectWorker from "../EffectWorker.ts"; +import * as Orchestrator from "../Orchestrator.ts"; +import * as ProviderAdapterRegistry from "../ProviderAdapterRegistry.ts"; +import { makeOrchestratorV2ReplayLayerWithRegistry } from "../testkit/ProviderReplayHarness.ts"; +import { describe } from "vite-plus/test"; +import { resolveAttachmentPath } from "../../attachmentStore.ts"; +import { KiloSessionError } from "../../provider/kilo/KiloSessionClient.ts"; +import * as KiloRuntime from "../../provider/kilo/KiloRuntime.ts"; +import * as IdAllocator from "../IdAllocator.ts"; +import type * as Adapter from "../ProviderAdapter.ts"; +import * as KiloTextGeneration from "../../textGeneration/KiloTextGeneration.ts"; +import * as KiloAdapter from "./KiloAdapterV2.ts"; + +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const binary = process.env.KILO_BIN; +const layer = Layer.mergeAll(NodeServices.layer, IdAllocator.layer); +const inference = Effect.acquireRelease( + Effect.promise(async () => { + const requests: Array> = []; + const control: { + mode: "text" | "approval" | "question" | "json" | "subagent" | "subagent-approval"; + } = { + mode: "text", + }; + const server = NodeHttp.createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => { + body += String(chunk); + }); + req.on("end", () => { + const parsed = JSON.parse(body) as Record; + requests.push(parsed); + if (parsed.stream !== true) { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + id: "chatcmpl-local", + object: "chat.completion", + created: 0, + model: "test", + choices: [ + { + index: 0, + message: { role: "assistant", content: "Local test" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }, + }), + ); + return; + } + res.writeHead(200, { "Content-Type": "text/event-stream" }); + const messages = parsed.messages as Array<{ role: string }>; + if ( + (control.mode === "subagent" || control.mode === "subagent-approval") && + messages.at(-1)?.role !== "tool" && + !JSON.stringify(messages.at(-1)).includes("Child fixture reply") + ) { + res.write( + `data: ${JSON.stringify({ + id: "chatcmpl-task", + object: "chat.completion.chunk", + created: 0, + model: "test", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + id: "call_task", + type: "function", + function: { + name: "task", + arguments: JSON.stringify({ + description: "Local child", + prompt: "Child fixture reply", + subagent_type: "general", + }), + }, + }, + ], + }, + finish_reason: null, + }, + ], + })}\n\n`, + ); + res.end( + `data: ${JSON.stringify({ + id: "chatcmpl-task", + object: "chat.completion.chunk", + created: 0, + model: "test", + choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], + })}\n\ndata: [DONE]\n\n`, + ); + return; + } + if ( + (control.mode === "approval" || + control.mode === "question" || + (control.mode === "subagent-approval" && + JSON.stringify(messages.at(-1)).includes("Child fixture reply"))) && + messages.at(-1)?.role !== "tool" + ) { + const name = control.mode === "question" ? "question" : "bash"; + const args = + control.mode !== "question" + ? { + command: "printf kilo-approved > approval.txt", + description: "Write the local approval fixture", + } + : { + questions: [ + { + question: "Choose a color", + header: "Color", + multiple: true, + + options: [ + { label: "Blue", description: "Blue option" }, + { label: "Red", description: "Red option" }, + ], + }, + ], + }; + res.write( + `data: ${JSON.stringify({ + id: "chatcmpl-tool", + object: "chat.completion.chunk", + created: 0, + model: "test", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + id: "call_fixture", + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }, + finish_reason: null, + }, + ], + })}\n\n`, + ); + res.end( + `data: ${JSON.stringify({ + id: "chatcmpl-tool", + object: "chat.completion.chunk", + created: 0, + model: "test", + choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], + })}\n\ndata: [DONE]\n\n`, + ); + return; + } + if (control.mode === "text") + res.write( + `data: ${JSON.stringify({ id: "chatcmpl-local", object: "chat.completion.chunk", created: 0, model: "test", choices: [{ index: 0, delta: { reasoning_content: "Fixture reasoning." }, finish_reason: null }] })}\n\n`, + ); + for (const text of control.mode === "json" + ? ['{"title":"Local fixture title"}'] + : ["Hello ", "from ", "local Kilo."]) + res.write( + `data: ${JSON.stringify({ + id: "chatcmpl-local", + object: "chat.completion.chunk", + created: 0, + model: "test", + choices: [{ index: 0, delta: { content: text }, finish_reason: null }], + })}\n\n`, + ); + res.end( + `data: ${JSON.stringify({ + id: "chatcmpl-local", + object: "chat.completion.chunk", + created: 0, + model: "test", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + })}\n\ndata: [DONE]\n\n`, + ); + }); + }); + server.listen(0, "127.0.0.1"); + await NodeEvents.EventEmitter.once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("No inference address"); + return { server, requests, control, url: `http://127.0.0.1:${address.port}/v1` }; + }), + ({ server }) => + Effect.promise(() => { + server.closeAllConnections(); + return new Promise((resolve) => server.close(() => resolve())); + }), +); + +describe.runIf(binary !== undefined)("Kilo adapter with native runtime and local inference", () => { + it.live( + "delivers a real streamed turn and restores its native history", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-adapter-" }); + const model = yield* inference; + const continuationKey = "account-scope"; + const instanceId = ProviderInstanceId.make("kilo-test"); + const threadId = ThreadId.make("kilo-thread"); + const modelSelection = { instanceId, model: "fixture/test", options: [] }; + const runtimePolicy = { + runtimeMode: "full-access" as const, + interactionMode: "default" as const, + cwd: root, + }; + const runtime = yield* KiloRuntime.make({ + instanceId: continuationKey, + binaryPath: binary!, + profileDirectory: path.join(root, "profile"), + environment: { + PATH: process.env.PATH, + HTTP_PROXY: process.env.HTTP_PROXY, + HTTPS_PROXY: process.env.HTTPS_PROXY, + NO_PROXY: process.env.NO_PROXY, + NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS, + KILO_DISABLE_AUTOUPDATE: "1", + KILO_DISABLE_MODELS_FETCH: "1", + KILO_DISABLE_DEFAULT_PLUGINS: "1", + KILO_DISABLE_EXTERNAL_SKILLS: "1", + KILO_DISABLE_PROJECT_CONFIG: "1", + KILO_CONFIG_CONTENT: encodeJson({ + model: "fixture/test", + small_model: "fixture/test", + plugin: [], + agent: { general: { permission: { bash: "ask" } } }, + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + name: "Local fixture", + options: { baseURL: model.url }, + models: { test: { name: "Test", limit: { context: 10000, output: 1000 } } }, + }, + }, + }), + }, + }); + let dropInteraction = false; + let droppedInteractions = 0; + // Fault injection drops the transport before T3 sees a real pending native request. + // All ownership checks, history, approvals and tool execution still use the actual CLI. + const disconnectedRuntime = KiloRuntime.KiloRuntime.of({ + open: (directory) => + runtime.open(directory).pipe( + Effect.map((connection) => ({ + ...connection, + client: { + ...connection.client, + events: (...args: Parameters) => + connection.client.events(...args).pipe( + Stream.mapEffect((event) => + Effect.suspend(() => { + if ( + dropInteraction && + (event.type === "permission.asked" || event.type === "question.asked") + ) { + dropInteraction = false; + droppedInteractions++; + return Effect.fail( + new KiloSessionError({ + operation: "event.subscribe", + reason: "request_failed", + }), + ); + } + return Effect.succeed(event); + }), + ), + ), + }, + })), + ), + }); + const adapter = yield* KiloAdapter.make({ + instanceId, + continuationKey, + cwd: root, + runtime: disconnectedRuntime, + attachmentsDir: path.join(root, "attachments"), + }); + const session = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("kilo-session"), + modelSelection, + runtimePolicy, + }); + const providerThread = yield* session.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + const seen: Adapter.ProviderAdapterV2Event[] = []; + const questionShown = yield* Deferred.make(); + let terminal = yield* Deferred.make(); + let interaction = + yield* Deferred.make< + Extract + >(); + yield* session.events.pipe( + Stream.runForEach((event) => { + seen.push(event); + if (event.type === "turn_item.updated" && event.turnItem.type === "user_input_request") + return Deferred.succeed(questionShown, undefined).pipe(Effect.asVoid); + if ( + event.type === "runtime_request.updated" && + event.runtimeRequest.status === "pending" + ) + return Deferred.succeed(interaction, event).pipe(Effect.asVoid); + return event.type === "turn.terminal" + ? Deferred.succeed(terminal, event).pipe(Effect.asVoid) + : Effect.void; + }), + Effect.forkScoped, + ); + const now = yield* DateTime.now; + const firstInput: Adapter.ProviderAdapterV2TurnInput = { + appThread: { + id: threadId, + projectId: ProjectId.make("kilo-project"), + title: "Local Kilo", + providerInstanceId: instanceId, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: providerThread.id, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + forkedFrom: null, + createdBy: "user", + creationSource: "web", + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }, + threadId, + runId: RunId.make("kilo-run"), + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: RunAttemptId.make("kilo-attempt"), + rootNodeId: NodeId.make("kilo-node"), + providerThread, + message: { + messageId: MessageId.make("kilo-message"), + text: "Say hello", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection, + runtimePolicy, + }; + yield* session.startTurn(firstInput); + const result = yield* Deferred.await(terminal); + assert.equal(result.type === "turn.terminal" ? result.status : undefined, "completed"); + const text = seen.findLast((event) => event.type === "message.updated"); + assert.equal( + text?.type === "message.updated" ? text.message.text : undefined, + "Hello from local Kilo.", + ); + assert.isAbove(model.requests.length, 0); + assert.isTrue( + seen.some((e) => e.type === "turn_item.updated" && e.turnItem.type === "reasoning"), + ); + const restored = yield* session.readThreadSnapshot({ providerThread }); + assert.equal(restored.messages.at(-1)?.text, "Hello from local Kilo."); + assert.deepEqual( + restored.messages.map((m) => m.role), + ["user", "assistant"], + ); + assert.equal(restored.providerTurns.length, 1); + assert.equal(restored.messages[0]!.id, firstInput.message.messageId); + const restoredSession = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("restored-session"), + modelSelection, + runtimePolicy, + }); + const restoredThread = yield* restoredSession.resumeThread({ + providerThread: restored.providerThread, + }); + const fromDisk = yield* restoredSession.readThreadSnapshot({ + providerThread: restoredThread, + }); + assert.equal(fromDisk.providerTurns.length, 1); + assert.equal(fromDisk.providerTurns[0]!.id, restored.providerTurns[0]!.id); + assert.equal(fromDisk.messages[0]!.id, firstInput.message.messageId); + assert.equal(fromDisk.messages[1]!.runId, firstInput.runId); + assert.equal( + DateTime.toEpochMillis(fromDisk.messages[0]!.createdAt), + DateTime.toEpochMillis(restored.messages[0]!.createdAt), + ); + + terminal = yield* Deferred.make(); + yield* session.startTurn({ + ...firstInput, + runId: RunId.make("second-run"), + runOrdinal: 2, + providerTurnOrdinal: 2, + attemptId: RunAttemptId.make("second-attempt"), + message: { + ...firstInput.message, + messageId: MessageId.make("second-message"), + text: "Say hello again", + }, + }); + // Delayed cancellation of turn one cannot stop turn two. + yield* session.interruptTurn({ + providerThread, + providerTurnId: restored.providerTurns[0]!.id, + }); + const second = yield* Deferred.await(terminal); + assert.equal(second.type === "turn.terminal" ? second.status : undefined, "completed"); + const secondHistory = yield* session.readThreadSnapshot({ providerThread }); + assert.equal(secondHistory.messages.length, 4); + const fork = yield* session.forkThread({ + sourceProviderThread: providerThread, + sourceProviderTurns: secondHistory.providerTurns, + providerTurnId: restored.providerTurns[0]!.id, + targetThreadId: ThreadId.make("fork-thread"), + }); + const forkSession = yield* adapter.openSession({ + threadId: ThreadId.make("fork-thread"), + providerSessionId: ProviderSessionId.make("fork-session"), + modelSelection, + runtimePolicy, + }); + const forkThread = yield* forkSession.resumeThread({ providerThread: fork }); + const forkHistory = yield* forkSession.readThreadSnapshot({ providerThread: forkThread }); + assert.deepEqual( + forkHistory.messages.map((m) => m.text), + restored.messages.map((m) => m.text), + ); + const rewound = yield* session.rollbackThread({ + providerThread, + providerThreadTurns: secondHistory.providerTurns, + target: { + type: "provider_turn", + providerTurn: restored.providerTurns[0]!, + appRunOrdinal: 1, + checkpointId: CheckpointId.make("rewind-checkpoint"), + }, + }); + assert.equal(rewound.messages.length, 2); + assert.equal(rewound.messages[0]!.id, firstInput.message.messageId); + assert.equal(rewound.messages[1]!.runId, firstInput.runId); + const rewindSession = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("rewind-resume"), + modelSelection, + runtimePolicy, + }); + const rewindThread = yield* rewindSession.resumeThread({ + providerThread: rewound.providerThread, + }); + const rewindDisk = yield* rewindSession.readThreadSnapshot({ + providerThread: rewindThread, + }); + assert.equal(rewindDisk.messages[0]!.id, firstInput.message.messageId); + assert.equal(rewindDisk.providerTurns[0]!.id, restored.providerTurns[0]!.id); + assert.notEqual( + rewound.providerTurns[0]!.nativeTurnRef?.nativeId, + restored.providerTurns[0]!.nativeTurnRef?.nativeId, + ); + // The source ref is stale after rewind and cannot mutate the replacement conversation. + yield* session.startTurn(firstInput).pipe(Effect.flip); + const empty = yield* session.rollbackThread({ + providerThread: rewound.providerThread, + providerThreadTurns: rewound.providerTurns, + target: { + type: "thread_start", + appRunOrdinal: 0, + checkpointId: CheckpointId.make("start-checkpoint"), + }, + }); + assert.equal(empty.messages.length, 0); + for (const decision of ["decline", "accept"] as const) { + terminal = yield* Deferred.make(); + interaction = + yield* Deferred.make< + Extract + >(); + model.control.mode = "approval"; + yield* session.startTurn({ + ...firstInput, + providerThread: empty.providerThread, + runtimePolicy: { ...runtimePolicy, runtimeMode: "approval-required" }, + message: { ...firstInput.message, text: "Run the approval fixture" }, + }); + const pending = yield* Deferred.await(interaction); + assert.equal(yield* fs.exists(path.join(root, "approval.txt")), false); + yield* session.respondToRuntimeRequest({ + requestId: pending.runtimeRequest.id, + decision, + }); + yield* Deferred.await(terminal); + assert.equal(yield* fs.exists(path.join(root, "approval.txt")), decision === "accept"); + } + terminal = yield* Deferred.make(); + interaction = + yield* Deferred.make< + Extract + >(); + dropInteraction = true; + model.control.mode = "question"; + yield* session.startTurn({ + ...firstInput, + providerThread: empty.providerThread, + message: { ...firstInput.message, text: "Ask the question fixture" }, + }); + const question = yield* Deferred.await(interaction); + assert.equal(question.runtimeRequest.kind, "user_input"); + assert.equal(droppedInteractions, 1); + yield* Deferred.await(questionShown); + const questionItem = seen.findLast( + (e) => e.type === "turn_item.updated" && e.turnItem.type === "user_input_request", + ); + assert.isTrue( + questionItem?.type === "turn_item.updated" && + questionItem.turnItem.type === "user_input_request" && + questionItem.turnItem.questions[0]?.multiSelect, + ); + // Kilo Question.Prompt permits multiple but native Info defaults custom to true. + assert.isTrue( + questionItem?.type === "turn_item.updated" && + questionItem.turnItem.type === "user_input_request" && + questionItem.turnItem.questions[0]?.allowCustomAnswer, + ); + yield* session.respondToRuntimeRequest({ + requestId: question.runtimeRequest.id, + answers: { "0": ["Blue", "Red"] }, + }); + yield* Deferred.await(terminal); + const requestNodes = new Map( + seen + .filter( + (event) => + event.type === "node.updated" && + (event.node.kind === "approval_request" || + event.node.kind === "user_input_request"), + ) + .map((event) => [event.type === "node.updated" ? event.node.id : "", event]), + ); + assert.isAbove(requestNodes.size, 0); + for (const event of requestNodes.values()) + assert.equal(event.type === "node.updated" ? event.node.status : undefined, "completed"); + model.control.mode = "subagent"; + terminal = yield* Deferred.make(); + yield* session.startTurn({ + ...firstInput, + providerThread: empty.providerThread, + message: { ...firstInput.message, text: "Delegate the local child fixture" }, + }); + yield* Deferred.await(terminal); + const children = seen.filter((event) => event.type === "subagent.updated"); + assert.isTrue(children.some((event) => event.subagent.status === "running")); + const child = children.findLast((event) => event.subagent.status === "completed"); + assert.isDefined(child?.subagent.childThreadId); + assert.isTrue( + seen.some( + (event) => + event.type === "message.updated" && + event.message.threadId === child?.subagent.childThreadId && + event.message.text.includes("Hello from local Kilo"), + ), + ); + for (const restrictedPolicy of [ + { ...runtimePolicy, runtimeMode: "approval-required" as const }, + { ...runtimePolicy, approvalPolicy: "on-request" }, + { ...runtimePolicy, interactionMode: "plan" as const }, + ]) { + terminal = yield* Deferred.make(); + const before = seen.length; + yield* session.startTurn({ + ...firstInput, + providerThread: empty.providerThread, + runtimePolicy: restrictedPolicy, + message: { + ...firstInput.message, + messageId: MessageId.make(`restricted-${before}`), + text: "Delegate the local child fixture", + }, + }); + yield* Deferred.await(terminal); + assert.isFalse(seen.slice(before).some((event) => event.type === "app_thread.created")); + } + model.control.mode = "json"; + const generated = yield* KiloTextGeneration.make(runtime).generateThreadTitle({ + cwd: root, + modelSelection, + message: "Name this test thread", + }); + assert.equal(generated.title, "Local fixture title"); + model.control.mode = "text"; + const workspace = path.join(root, "integration-workspace"); + yield* fs.makeDirectory(workspace); + yield* fs.writeFileString(path.join(workspace, "README.md"), "Kilo integration fixture\n"); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + for (const args of [ + ["init"], + ["add", "README.md"], + [ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-m", + "fixture", + ], + ]) { + const process = yield* spawner.spawn(ChildProcess.make("git", args, { cwd: workspace })); + assert.equal(Number(yield* process.exitCode), 0); + } + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const appThreadId = ThreadId.make("orchestrated-kilo-thread"); + const done = yield* Deferred.make(); + yield* orchestrator.streamStoredEvents.pipe( + Stream.runForEach(({ event }) => + event.threadId === appThreadId && + event.type === "run.updated" && + ["completed", "failed", "interrupted"].includes(event.payload.status) + ? Deferred.succeed(done, undefined).pipe(Effect.asVoid) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("kilo-create"), + createdBy: "user", + creationSource: "web", + threadId: appThreadId, + projectId: ProjectId.make("kilo-orchestration-project"), + title: "Kilo integrated", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: workspace, + }); + const attachment = { + type: "file" as const, + id: ChatAttachmentId.make("kilo-fixture"), + name: "fixture.txt", + mimeType: "text/plain", + sizeBytes: 7, + }; + const attachmentPath = resolveAttachmentPath({ + attachmentsDir: path.join(root, "attachments"), + attachment, + }); + if (!attachmentPath) throw new Error("Invalid fixture attachment path"); + yield* fs.makeDirectory(path.dirname(attachmentPath), { recursive: true }); + yield* fs.writeFileString(attachmentPath, "fixture"); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("kilo-send"), + createdBy: "user", + creationSource: "web", + threadId: appThreadId, + messageId: MessageId.make("orchestrated-user-message"), + text: "Hello through Orchestrator V2", + attachments: [attachment], + modelSelection, + dispatchMode: { type: "start_immediately" }, + }); + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + const startedProjection = yield* orchestrator.getThreadProjection(appThreadId); + assert.isAbove(startedProjection.runs.length, 0, encodeJson(startedProjection)); + yield* Deferred.await(done); + const projection = yield* orchestrator.getThreadProjection(appThreadId); + assert.equal(projection.runs[0]?.status, "completed"); + assert.equal(projection.providerThreads.length, 1); + const userMessage = projection.messages.find((m) => m.id === "orchestrated-user-message"); + assert.deepEqual(userMessage?.attachments, [attachment]); + assert.equal(userMessage?.text, "Hello through Orchestrator V2"); + assert.equal(userMessage?.creationSource, "web"); + assert.deepEqual( + projection.messages.filter((m) => m.role === "user").map((m) => m.id), + ["orchestrated-user-message"], + ); + assert.isTrue( + projection.turnItems.some( + (item) => + item.type === "assistant_message" && + item.text === "Hello from local Kilo." && + !item.streaming, + ), + ); + }).pipe( + Effect.provide( + makeOrchestratorV2ReplayLayerWithRegistry( + { name: "kilo-native-integration" }, + ProviderAdapterRegistry.makeSingleLayer(adapter), + ), + ), + ); + for (const action of ["accept", "stop"] as const) { + yield* fs.remove(path.join(root, "approval.txt"), { force: true }); + dropInteraction = action === "accept"; + model.control.mode = "subagent-approval"; + terminal = yield* Deferred.make(); + interaction = + yield* Deferred.make< + Extract + >(); + const startIndex = seen.length; + yield* session.startTurn({ + ...firstInput, + providerThread: empty.providerThread, + runtimePolicy, + message: { + ...firstInput.message, + messageId: MessageId.make(`child-${action}`), + text: "Delegate the local child fixture", + }, + }); + const pending = yield* Effect.raceFirst( + Deferred.await(interaction), + Deferred.await(terminal).pipe( + Effect.flatMap((event) => + Effect.die( + new Error( + encodeJson({ unexpectedTerminal: event, events: seen.slice(startIndex) }), + ), + ), + ), + ), + ); + assert.equal(pending.runtimeRequest.kind, "command"); + assert.equal(droppedInteractions, 2); + assert.isFalse(yield* fs.exists(path.join(root, "approval.txt"))); + const started = seen + .slice(startIndex) + .find((e) => e.type === "provider_turn.updated" && e.providerTurn.status === "running"); + if (started?.type !== "provider_turn.updated") throw new Error("No active native turn"); + if (action === "accept") { + yield* session.respondToRuntimeRequest({ + requestId: pending.runtimeRequest.id, + decision: "accept", + }); + } else { + yield* session.interruptTurn({ + providerThread: empty.providerThread, + providerTurnId: started.providerTurn.id, + }); + } + const ended = yield* Deferred.await(terminal); + assert.equal( + ended.type === "turn.terminal" ? ended.status : undefined, + action === "accept" ? "completed" : "interrupted", + ); + assert.equal(yield* fs.exists(path.join(root, "approval.txt")), action === "accept"); + if (action === "stop") { + const childStates = new Map( + seen + .slice(startIndex) + .filter((e) => e.type === "subagent.updated") + .map((e) => [e.subagent.id, e.subagent.status]), + ); + assert.isAbove(childStates.size, 0); + assert.isTrue([...childStates.values()].every((status) => status !== "running")); + yield* session + .respondToRuntimeRequest({ requestId: pending.runtimeRequest.id, decision: "accept" }) + .pipe(Effect.flip); + yield* session + .startTurn({ ...firstInput, providerThread: empty.providerThread }) + .pipe(Effect.flip); + const saved = seen + .slice(startIndex) + .findLast( + (e) => + e.type === "provider_thread.updated" && + e.providerThread.id === empty.providerThread.id, + ); + if (saved?.type !== "provider_thread.updated") + throw new Error("Missing durable interruption metadata"); + const fresh = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("after-stop"), + modelSelection, + runtimePolicy, + }); + const resumed = yield* fresh.resumeThread({ providerThread: saved.providerThread }); + const history = yield* fresh.readThreadSnapshot({ providerThread: resumed }); + assert.equal( + history.providerTurns.find((turn) => turn.id === started.providerTurn.id)?.status, + "interrupted", + ); + const invalidAgentDone = yield* Deferred.make(); + yield* fresh.events.pipe( + Stream.runForEach((event) => + event.type === "turn.terminal" + ? Deferred.succeed(invalidAgentDone, event).pipe(Effect.asVoid) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* fresh.startTurn({ + ...firstInput, + providerThread: resumed, + modelSelection: { + ...modelSelection, + options: [{ id: "agent", value: "missing-kilo-fixture-agent" }], + }, + message: { + ...firstInput.message, + messageId: MessageId.make("missing-agent-message"), + text: "Fail before creating an assistant", + }, + }); + const invalidAgent = yield* Deferred.await(invalidAgentDone); + assert.equal( + invalidAgent.type === "turn.terminal" ? invalidAgent.status : undefined, + "failed", + ); + } + } + const otherAccount = yield* KiloAdapter.make({ + instanceId, + continuationKey: "different-account", + cwd: root, + runtime, + }); + const otherSession = yield* otherAccount.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("other"), + modelSelection, + runtimePolicy, + }); + yield* otherSession.resumeThread({ providerThread }).pipe(Effect.flip); + yield* otherSession + .ensureThread({ + threadId, + modelSelection, + runtimePolicy, + existingProviderThread: { ...restored.providerThread, nativeThreadRef: null }, + }) + .pipe(Effect.flip); + }).pipe(Effect.scoped, Effect.provide(layer)), + { timeout: 120000 }, + ); +}); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts new file mode 100644 index 000000000000..3fcce44db8a7 --- /dev/null +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts @@ -0,0 +1,1677 @@ +import type { Event, Part, PermissionRequest, QuestionRequest } from "@kilocode/sdk/v2"; +import { + ProviderDriverKind, + RuntimeRequestId, + type OrchestrationV2ExecutionNode, + type OrchestrationV2Subagent, + type ProviderInstanceId, + type OrchestrationV2ProviderCapabilities, + type OrchestrationV2ProviderThread, + type OrchestrationV2ProviderTurn, + type OrchestrationV2ConversationMessage, + type OrchestrationV2RuntimeRequest, + type OrchestrationV2TurnItem, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Queue from "effect/Queue"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; + +import { resolveAttachmentPath } from "../../attachmentStore.ts"; +import { toOpenCodeFileParts } from "../../provider/opencodeRuntime.ts"; +import * as KiloRuntime from "../../provider/kilo/KiloRuntime.ts"; +import { KiloSessionError, type KiloSessionRef } from "../../provider/kilo/KiloSessionClient.ts"; +import * as IdAllocator from "../IdAllocator.ts"; +import * as Adapter from "../ProviderAdapter.ts"; +import { turnScopedSelectionTransition } from "../ProviderSelectionTransition.ts"; +import { makeProviderFailure } from "../ProviderFailure.ts"; +import { + makeSubagentChildThread, + makeSubagentConversationArtifacts, +} from "../SubagentProjection.ts"; +import { openCodePermissionRules } from "./OpenCodeAdapterV2.ts"; +import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; +import { openCodeToolTurnItem } from "./OpenCodeToolItems.ts"; + +export const KILO_PROVIDER = ProviderDriverKind.make("kilo"); + +const capabilities: OrchestrationV2ProviderCapabilities = { + sessions: { + supportsMultipleProviderThreadsPerSession: false, + supportsModelSwitchInSession: true, + supportsProviderSwitchingViaHandoff: false, + supportsRuntimeModeSwitchInSession: false, + pendingRequestsSurviveRestart: false, + }, + threads: { + canCreateEmptyThread: true, + canReadThreadSnapshot: true, + canRollbackThread: true, + canForkThread: true, + canForkFromTurn: true, + canForkFromSubagentThread: false, + exposesNativeThreadId: true, + }, + turns: { + exposesNativeTurnId: false, + emitsTurnStarted: true, + emitsTurnCompleted: true, + supportsInterrupt: true, + supportsActiveSteering: false, + supportsSteeringByInterruptRestart: false, + supportsQueuedMessages: false, + terminalStatusQuality: "strong", + }, + streaming: { + streamsAssistantText: true, + streamsReasoning: true, + streamsToolOutput: false, + streamsPlanText: false, + emitsMessageCompleted: true, + }, + tools: { + exposesToolItemIds: true, + emitsToolStarted: true, + emitsToolCompleted: true, + emitsToolOutput: true, + supportsMcpTools: false, + supportsDynamicToolCallbacks: false, + }, + approvals: { + supportsCommandApproval: true, + supportsFileReadApproval: true, + supportsFileChangeApproval: true, + supportsApplyPatchApproval: true, + approvalsHaveNativeRequestIds: true, + approvalCallbacksAreLiveOnly: true, + approvalsCanOriginateFromSubagents: true, + }, + planning: { + emitsPlanUpdated: false, + emitsTodoList: false, + emitsProposedPlan: false, + supportsStructuredQuestions: true, + planDeltasHaveItemIds: false, + }, + subagents: { + supportsSubagents: true, + exposesSubagentThreadIds: true, + emitsSubagentLifecycle: true, + canWaitForSubagents: true, + canCloseSubagents: false, + canForkSubagentThread: false, + }, + context: { + acceptsSystemContext: false, + acceptsDeveloperContext: false, + acceptsSyntheticUserContext: false, + canGenerateSummaries: false, + canConsumeHandoffSummaries: false, + supportsDeltaHandoff: false, + supportsFullThreadHandoff: false, + maxRecommendedHandoffChars: null, + }, + checkpointing: { + appCanCheckpointFilesystem: true, + supportsNestedCheckpointScopes: false, + providerCanRollbackConversation: true, + providerRollbackReturnsSnapshot: true, + providerCanReadConversationSnapshot: true, + }, + identity: { + nativeThreadIds: "strong", + nativeTurnIds: "weak", + nativeItemIds: "strong", + nativeRequestIds: "strong", + }, + runtimePolicy: { enforcement: "native" }, +}; + +const error = (detail: string) => + new Adapter.ProviderAdapterProtocolError({ driver: KILO_PROVIDER, detail }); +const nativeRef = (nativeId: string) => ({ + driver: KILO_PROVIDER, + nativeId, + strength: "strong" as const, +}); +const wire = (effect: Effect.Effect) => + effect.pipe(Effect.mapError(() => error("Kilo request failed; the operation was not retried"))); + +// Kilo 7.8.3 task.ts persists inherited edit/bash/MCP ceilings before launching a child. +const permissions = (policy: Adapter.ProviderAdapterV2RuntimePolicy) => { + const rules = openCodePermissionRules(policy); + // Kilo re-appends session denies in Plan and inherits them into children. OpenCode's + // temporary deny seeds would therefore override later ask rules permanently. + const effective = rules.filter( + (rule, index) => + !rules + .slice(index + 1) + .some((later) => later.permission === rule.permission && later.pattern === rule.pattern), + ); + // Kilo 7.8.3 inherits denies, not asks, into child sessions and provides no pre-start + // child policy hook. Do not allow a child to escape the selected T3 approval policy. + const unrestricted = + effective.length === 1 && + effective[0]?.permission === "*" && + effective[0]?.pattern === "*" && + effective[0]?.action === "allow"; + if (!unrestricted || policy.interactionMode !== "default") + effective.push({ permission: "task", pattern: "*", action: "deny" }); + return effective; +}; + +export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { + readonly instanceId: ProviderInstanceId; + readonly continuationKey: string; + readonly cwd: string; + readonly attachmentsDir?: string; + readonly runtime: KiloRuntime.KiloRuntime["Service"]; +}) { + const ids = yield* IdAllocator.IdAllocatorV2; + const crypto = yield* Crypto.Crypto; + const itemKey = (nativeId: string) => + `${options.instanceId}:${options.continuationKey}:${nativeId}`; + return Adapter.ProviderAdapterV2.of({ + instanceId: options.instanceId, + driver: KILO_PROVIDER, + getCapabilities: () => Effect.succeed(capabilities), + planSelectionTransition: () => Effect.succeed(turnScopedSelectionTransition()), + openSession: Effect.fn("KiloAdapterV2.openSession")(function* (input) { + const scope = yield* Effect.scope; + const turnGate = yield* Semaphore.make(1); + const directory = input.runtimePolicy.cwd ?? options.cwd; + const connection = yield* wire(options.runtime.open(directory)); + const client = connection.client; + const now = yield* DateTime.now; + const session = { + id: input.providerSessionId, + driver: KILO_PROVIDER, + providerInstanceId: options.instanceId, + cwd: directory, + model: input.modelSelection.model, + status: "ready" as const, + capabilities, + createdAt: now, + updatedAt: now, + lastError: null, + }; + const events = yield* Queue.unbounded(); + const nodes = new Map(); + const items = new Map(); + const emit = (event: Adapter.ProviderAdapterV2Event) => + Effect.suspend(() => { + if (event.type === "node.updated") nodes.set(event.node.id, event.node); + if (event.type === "turn_item.updated") items.set(event.turnItem.id, event.turnItem); + return Queue.offer(events, event).pipe(Effect.asVoid); + }); + let ref: KiloSessionRef | undefined; + let thread: OrchestrationV2ProviderThread | undefined; + let active: + | { + input: Adapter.ProviderAdapterV2TurnInput; + turn: OrchestrationV2ProviderTurn; + messageID: string; + admitted: boolean; + interrupting: boolean; + reportedError?: boolean; + finishing?: boolean; + } + | undefined; + const messages = new Map(); + const parts = new Map(); + const partsByMessage = new Map>(); + const timestamps = new Map(); + const putPart = (part: Part) => { + parts.set(part.id, part); + const grouped = partsByMessage.get(part.messageID) ?? new Map(); + grouped.set(part.id, part); + partsByMessage.set(part.messageID, grouped); + }; + const roles = new Map(); + const parents = new Map(); + const completedMessages = new Set(); + const providerTurns = new Map(); + const childThreads = new Map(); + const subagents = new Map(); + const requests = new Map< + RuntimeRequestId, + { runtime: OrchestrationV2RuntimeRequest; native: PermissionRequest | QuestionRequest } + >(); + const ordinals = new Map(); + const ordinal = (key: string) => { + let value = ordinals.get(key); + if (value === undefined) { + value = ordinals.size + 1; + ordinals.set(key, value); + } + return value; + }; + const current = () => + ref === undefined ? Effect.fail(error("Kilo thread is not loaded")) : Effect.succeed(ref); + const ownedThread = (candidate: OrchestrationV2ProviderThread) => + thread && + candidate.id === thread.id && + candidate.appThreadId === thread.appThreadId && + candidate.driver === KILO_PROVIDER && + candidate.providerInstanceId === options.instanceId && + candidate.nativeThreadRef?.nativeId === ref?.sessionId && + candidate.nativeMetadata?.continuationKey === options.continuationKey + ? current() + : Effect.fail(error("Kilo thread does not belong to this account and session")); + const resolveMessage = (id: string) => thread?.nativeMetadata?.messageAliases?.[id] ?? id; + let correlationMetadata: OrchestrationV2ProviderThread["nativeMetadata"]; + const correlatedMessages = new Map< + string, + NonNullable< + NonNullable["turnCorrelations"] + >[string] + >(); + const correlationFor = (id: string) => { + if (correlationMetadata !== thread?.nativeMetadata) { + correlationMetadata = thread?.nativeMetadata; + correlatedMessages.clear(); + for (const [nativeId, value] of Object.entries( + correlationMetadata?.turnCorrelations ?? {}, + )) + correlatedMessages.set(resolveMessage(nativeId), value); + } + return correlatedMessages.get(id); + }; + const finish = Effect.fn("KiloAdapterV2.finish")(function* ( + status: "completed" | "interrupted" | "failed", + detail?: string, + ) { + const running = active; + if (!running || running.finishing) return; + running.finishing = true; + const completedAt = yield* DateTime.now; + providerTurns.set(running.turn.id, { ...running.turn, status, completedAt }); + const correlation = thread?.nativeMetadata?.turnCorrelations?.[running.messageID]; + if (thread && correlation) { + thread = { + ...thread, + nativeMetadata: { + ...thread.nativeMetadata, + turnCorrelations: { + ...thread.nativeMetadata?.turnCorrelations, + [running.messageID]: { + ...correlation, + terminalStatus: status, + completedAt: DateTime.formatIso(completedAt), + }, + }, + }, + }; + yield* emit({ + type: "provider_thread.updated", + driver: KILO_PROVIDER, + providerThread: thread, + }); + } + for (const pending of requests.values()) { + if (pending.runtime.status !== "pending") continue; + pending.runtime = { ...pending.runtime, status: "cancelled", resolvedAt: completedAt }; + yield* emit({ + type: "runtime_request.updated", + driver: KILO_PROVIDER, + runtimeRequest: pending.runtime, + }); + } + for (const node of nodes.values()) { + if ( + node.providerTurnId !== running.turn.id || + !["running", "waiting"].includes(node.status) + ) + continue; + yield* emit({ + type: "node.updated", + driver: KILO_PROVIDER, + node: { ...node, status, completedAt }, + }); + } + for (const item of items.values()) { + if ( + item.providerTurnId !== running.turn.id || + !["running", "waiting"].includes(item.status) + ) + continue; + yield* emit({ + type: "turn_item.updated", + driver: KILO_PROVIDER, + turnItem: { + ...item, + status, + completedAt, + updatedAt: completedAt, + ...("streaming" in item ? { streaming: false } : {}), + }, + }); + } + for (const [id, message] of messages) { + if (!message.streaming) continue; + const completed = { ...message, streaming: false, updatedAt: completedAt }; + messages.set(id, completed); + yield* emit({ type: "message.updated", driver: KILO_PROVIDER, message: completed }); + } + for (const [id, child] of subagents) { + if (child.runId !== running.input.runId || child.status !== "running") continue; + const ended = { ...child, status, completedAt, updatedAt: completedAt }; + subagents.set(id, ended); + yield* emit({ type: "subagent.updated", driver: KILO_PROVIDER, subagent: ended }); + } + for (const [id, child] of childThreads) { + if (child.status !== "active") continue; + const ended = { ...child, status: "idle" as const, updatedAt: completedAt }; + childThreads.set(id, ended); + yield* emit({ + type: "provider_thread.updated", + driver: KILO_PROVIDER, + providerThread: ended, + }); + } + active = undefined; + yield* emit({ + type: "provider_turn.updated", + driver: KILO_PROVIDER, + providerTurn: { ...running.turn, status, completedAt }, + }); + if (status === "failed") + yield* emit({ + type: "turn.terminal", + driver: KILO_PROVIDER, + providerThreadId: running.turn.providerThreadId, + providerTurnId: running.turn.id, + runOrdinal: running.input.runOrdinal, + failureItemOrdinal: ordinals.size + 1, + status, + failure: makeProviderFailure({ + message: detail ?? "Kilo failed", + class: "provider_error", + }), + threadDisposition: "broken", + }); + else + yield* emit({ + type: "turn.terminal", + driver: KILO_PROVIDER, + providerThreadId: running.turn.providerThreadId, + providerTurnId: running.turn.id, + runOrdinal: running.input.runOrdinal, + status, + failure: null, + threadDisposition: status === "interrupted" ? "broken" : "reusable", + }); + }); + yield* connection.exitCode.pipe( + Effect.andThen( + Effect.gen(function* () { + // The Stop owner publishes terminality only after the entire owned process group is gone. + if (!active || active.interrupting) return; + yield* connection.cleanup; + yield* finish("failed", "Kilo process exited."); + }), + ), + Effect.forkIn(scope), + ); + const messageFromParts = Effect.fn("KiloAdapterV2.message")(function* ( + messageID: string, + completed = false, + publish = true, + ) { + if (!thread?.appThreadId || !roles.has(messageID)) return; + const role = roles.get(messageID)!; + const updatedAt = yield* DateTime.now; + const owningTurn = + messageID === active?.messageID || parents.get(messageID) === active?.messageID + ? active + : undefined; + const correlationId = role === "user" ? messageID : (parents.get(messageID) ?? ""); + const correlation = correlationFor(correlationId); + const text = [...(partsByMessage.get(messageID)?.values() ?? [])] + .filter((part) => part.type === "text") + .map((part) => (part.type === "text" ? part.text : "")) + .join(""); + const previous = messages.get(messageID); + const message: OrchestrationV2ConversationMessage = { + id: + (role === "user" ? correlation?.messageId : undefined) ?? + ids.derive.messageFromProviderItem({ + driver: KILO_PROVIDER, + nativeItemId: itemKey(messageID), + }), + threadId: thread.appThreadId, + runId: owningTurn?.input.runId ?? previous?.runId ?? correlation?.runId ?? null, + nodeId: owningTurn?.input.rootNodeId ?? previous?.nodeId ?? correlation?.nodeId ?? null, + role, + text, + attachments: role === "user" ? (correlation?.attachments ?? []) : [], + streaming: role === "assistant" && !completed && !completedMessages.has(messageID), + createdAt: previous?.createdAt ?? timestamps.get(messageID) ?? updatedAt, + updatedAt, + createdBy: role === "user" ? (correlation?.createdBy ?? "user") : "agent", + creationSource: + role === "user" ? (correlation?.creationSource ?? "provider") : "provider", + ...(role === "user" + ? { + scheduledTaskId: correlation?.scheduledTaskId, + senderThreadId: correlation?.senderThreadId, + } + : {}), + }; + messages.set(messageID, message); + // T3 owns submitted user messages, including context and provenance absent from native history. + // Echoing native file/text parts must not overwrite that durable row. + if (publish && !(role === "user" && correlation)) + yield* emit({ type: "message.updated", driver: KILO_PROVIDER, message }); + }); + const textPart = Effect.fn("KiloAdapterV2.textPart")(function* ( + part: Extract, + ) { + const running = active; + if (!running || !thread || parents.get(part.messageID) !== running.messageID) return; + const at = yield* DateTime.now; + const done = part.time?.end !== undefined || completedMessages.has(part.messageID); + const key = itemKey(part.id); + const nodeId = ids.derive.nodeFromProviderItem({ + driver: KILO_PROVIDER, + nativeItemId: key, + }); + const base = { + id: ids.derive.turnItemFromProviderItem({ driver: KILO_PROVIDER, nativeItemId: key }), + threadId: running.input.threadId, + runId: running.input.runId, + nodeId, + providerThreadId: thread.id, + providerTurnId: running.turn.id, + nativeItemRef: nativeRef(part.id), + parentItemId: null, + ordinal: ordinal(part.id), + status: done ? ("completed" as const) : ("running" as const), + title: null, + startedAt: running.turn.startedAt, + completedAt: done ? at : null, + updatedAt: at, + }; + yield* emit({ + type: "node.updated", + driver: KILO_PROVIDER, + node: { + id: nodeId, + threadId: running.input.threadId, + runId: running.input.runId, + parentNodeId: running.input.rootNodeId, + rootNodeId: running.input.rootNodeId, + kind: part.type === "text" ? "assistant_message" : "reasoning", + status: base.status, + countsForRun: false, + providerThreadId: thread.id, + providerTurnId: running.turn.id, + nativeItemRef: nativeRef(part.id), + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: base.startedAt, + completedAt: base.completedAt, + }, + }); + yield* emit({ + type: "turn_item.updated", + driver: KILO_PROVIDER, + turnItem: + part.type === "reasoning" + ? { ...base, type: "reasoning", text: part.text, streaming: !done } + : { + ...base, + type: "assistant_message", + messageId: ids.derive.messageFromProviderItem({ + driver: KILO_PROVIDER, + nativeItemId: itemKey(part.messageID), + }), + text: part.text, + streaming: !done, + }, + }); + }); + const task = Effect.fn("KiloAdapterV2.task")(function* ( + part: Extract, + ) { + const running = active; + if (!running || !thread) return; + const at = yield* DateTime.now; + const key = itemKey(part.id); + const nodeId = ids.derive.nodeFromProviderItem({ + driver: KILO_PROVIDER, + nativeItemId: key, + }); + const metadata = + part.state.status === "running" || part.state.status === "completed" + ? part.state.metadata + : undefined; + const childId = typeof metadata?.sessionId === "string" ? metadata.sessionId : undefined; + const prompt = typeof part.state.input.prompt === "string" ? part.state.input.prompt : ""; + const title = + typeof part.state.input.description === "string" + ? part.state.input.description + : "Kilo subagent"; + const previous = subagents.get(part.id); + let child = childId + ? childThreads.get(childId) + : [...childThreads.values()].find( + (candidate) => candidate.id === previous?.providerThreadId, + ); + if (childId && !child) { + const native = { instanceId: options.continuationKey, directory, sessionId: childId }; + const nativeChild = yield* wire(client.read(native)); + if (nativeChild.parentID !== ref?.sessionId) + return yield* error("Kilo task is not owned by this conversation"); + const childThreadId = ids.derive.threadFromProviderThread({ + driver: KILO_PROVIDER, + nativeThreadId: itemKey(childId), + }); + child = { + ...thread, + id: ids.derive.providerThread({ + driver: KILO_PROVIDER, + providerInstanceId: options.instanceId, + nativeThreadId: itemKey(childId), + }), + appThreadId: childThreadId, + ownerNodeId: nodeId, + nativeThreadRef: nativeRef(childId), + status: "active", + nativeMetadata: { continuationKey: options.continuationKey, itemIdentityVersion: 2 }, + }; + childThreads.set(childId, child); + yield* emit({ + type: "app_thread.created", + driver: KILO_PROVIDER, + appThread: makeSubagentChildThread({ + parentThread: running.input.appThread, + childThreadId, + parentNodeId: nodeId, + activeProviderThreadId: child.id, + providerInstanceId: options.instanceId, + modelSelection: running.input.modelSelection, + title, + now: at, + createdBy: "agent", + creationSource: "provider", + }), + }); + yield* emit({ + type: "provider_thread.updated", + driver: KILO_PROVIDER, + providerThread: child, + }); + } + const status = + part.state.status === "completed" + ? "completed" + : part.state.status === "error" + ? "failed" + : "running"; + const completedAt = status === "running" ? null : at; + const subagent: OrchestrationV2Subagent = { + id: nodeId, + threadId: running.input.threadId, + runId: running.input.runId, + parentNodeId: running.input.rootNodeId, + origin: "provider_native", + createdBy: "agent", + driver: KILO_PROVIDER, + providerInstanceId: options.instanceId, + providerThreadId: child?.id ?? previous?.providerThreadId ?? null, + childThreadId: child?.appThreadId ?? previous?.childThreadId ?? null, + nativeTaskRef: nativeRef(part.id), + prompt, + title, + model: null, + status, + result: part.state.status === "completed" ? part.state.output : null, + startedAt: previous?.startedAt ?? at, + completedAt, + updatedAt: at, + }; + subagents.set(part.id, subagent); + yield* emit({ type: "subagent.updated", driver: KILO_PROVIDER, subagent }); + yield* emit({ + type: "node.updated", + driver: KILO_PROVIDER, + node: { + id: nodeId, + threadId: running.input.threadId, + runId: running.input.runId, + parentNodeId: running.input.rootNodeId, + rootNodeId: running.input.rootNodeId, + kind: "subagent", + status, + countsForRun: false, + providerThreadId: child?.id ?? thread.id, + providerTurnId: running.turn.id, + nativeItemRef: nativeRef(part.id), + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: subagent.startedAt, + completedAt, + }, + }); + yield* emit({ + type: "turn_item.updated", + driver: KILO_PROVIDER, + turnItem: { + id: ids.derive.turnItemFromProviderItem({ driver: KILO_PROVIDER, nativeItemId: key }), + threadId: running.input.threadId, + runId: running.input.runId, + nodeId, + providerThreadId: thread.id, + providerTurnId: running.turn.id, + nativeItemRef: nativeRef(part.id), + parentItemId: null, + ordinal: ordinal(part.id), + status, + title, + startedAt: subagent.startedAt, + completedAt, + updatedAt: at, + type: "subagent", + subagentId: nodeId, + origin: "provider_native", + driver: KILO_PROVIDER, + providerInstanceId: options.instanceId, + childThreadId: subagent.childThreadId, + prompt, + result: subagent.result, + }, + }); + const completedChildId = child?.nativeThreadRef?.nativeId; + if (completedChildId && child?.appThreadId && completedAt) { + const history = yield* wire( + client.history({ + instanceId: options.continuationKey, + directory, + sessionId: completedChildId, + }), + ); + for (const [index, entry] of history.entries()) { + const text = entry.parts + .filter((p) => p.type === "text") + .map((p) => p.text) + .join("\n"); + const artifacts = makeSubagentConversationArtifacts({ + messageId: ids.derive.messageFromProviderItem({ + driver: KILO_PROVIDER, + nativeItemId: itemKey(entry.info.id), + }), + turnItemId: ids.derive.turnItemFromProviderItem({ + driver: KILO_PROVIDER, + nativeItemId: itemKey(entry.info.id), + }), + threadId: child.appThreadId, + rootNodeId: nodeId, + providerThreadId: child.id, + providerTurnId: null, + nativeItemRef: nativeRef(entry.info.id), + role: entry.info.role, + text, + ordinal: index + 1, + now: DateTime.makeUnsafe(entry.info.time.created), + }); + yield* emit({ + type: "message.updated", + driver: KILO_PROVIDER, + message: artifacts.message, + }); + yield* emit({ + type: "turn_item.updated", + driver: KILO_PROVIDER, + turnItem: artifacts.turnItem, + }); + } + const endedChild = { ...child, status: "idle" as const, updatedAt: at }; + childThreads.set(completedChildId, endedChild); + yield* emit({ + type: "provider_thread.updated", + driver: KILO_PROVIDER, + providerThread: endedChild, + }); + } + }); + const tool = Effect.fn("KiloAdapterV2.tool")(function* ( + part: Extract, + ) { + if (part.tool === "task") return yield* task(part); + if (part.tool === "question") return; + const running = active; + if (!running || !thread || parents.get(part.messageID) !== running.messageID) return; + const at = yield* DateTime.now; + const key = itemKey(part.id); + const nodeId = ids.derive.nodeFromProviderItem({ + driver: KILO_PROVIDER, + nativeItemId: key, + }); + const done = part.state.status === "completed" || part.state.status === "error"; + const status = part.state.status === "error" ? "failed" : part.state.status; + const base = { + id: ids.derive.turnItemFromProviderItem({ driver: KILO_PROVIDER, nativeItemId: key }), + threadId: running.input.threadId, + runId: running.input.runId, + nodeId, + providerThreadId: thread.id, + providerTurnId: running.turn.id, + nativeItemRef: nativeRef(part.id), + parentItemId: null, + ordinal: ordinal(part.id), + status: status as OrchestrationV2TurnItem["status"], + title: part.tool, + startedAt: running.turn.startedAt, + completedAt: done ? at : null, + updatedAt: at, + }; + yield* emit({ + type: "node.updated", + driver: KILO_PROVIDER, + node: { + id: nodeId, + threadId: running.input.threadId, + runId: running.input.runId, + parentNodeId: running.input.rootNodeId, + rootNodeId: running.input.rootNodeId, + kind: "tool_call", + status, + countsForRun: false, + providerThreadId: thread.id, + providerTurnId: running.turn.id, + nativeItemRef: nativeRef(part.id), + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: running.turn.startedAt, + completedAt: done ? at : null, + }, + }); + yield* emit({ + type: "turn_item.updated", + driver: KILO_PROVIDER, + turnItem: openCodeToolTurnItem(base, { + name: part.tool, + input: part.state.input, + output: + part.state.status === "completed" + ? part.state.output + : part.state.status === "error" + ? part.state.error + : undefined, + completedMetadata: part.state.status === "completed" ? part.state.metadata : undefined, + }), + }); + }); + const ask = Effect.fn("KiloAdapterV2.ask")(function* ( + native: PermissionRequest | QuestionRequest, + ) { + const running = active; + if (!running || !thread || requests.has(RuntimeRequestId.make(itemKey(native.id)))) return; + const at = yield* DateTime.now; + const requestId = RuntimeRequestId.make(itemKey(native.id)); + const nodeId = ids.derive.approvalNode({ requestId }); + const question = "questions" in native; + const kind = question + ? "user_input" + : /edit|write|patch/.test(native.permission) + ? "file-change" + : /read|glob|grep/.test(native.permission) + ? "file-read" + : "command"; + const runtime: OrchestrationV2RuntimeRequest = { + id: requestId, + nodeId, + providerTurnId: running.turn.id, + nativeRequestRef: nativeRef(native.id), + kind, + status: "pending", + responseCapability: { type: "live", providerSessionId: input.providerSessionId }, + createdAt: at, + resolvedAt: null, + }; + requests.set(requestId, { runtime, native }); + yield* emit({ + type: "node.updated", + driver: KILO_PROVIDER, + node: { + id: nodeId, + threadId: running.input.threadId, + runId: running.input.runId, + parentNodeId: running.input.rootNodeId, + rootNodeId: running.input.rootNodeId, + kind: question ? "user_input_request" : "approval_request", + status: "waiting", + countsForRun: false, + providerThreadId: thread.id, + providerTurnId: running.turn.id, + nativeItemRef: nativeRef(native.id), + runtimeRequestId: requestId, + checkpointScopeId: null, + startedAt: at, + completedAt: null, + }, + }); + yield* emit({ + type: "runtime_request.updated", + driver: KILO_PROVIDER, + threadId: running.input.threadId, + runtimeRequest: runtime, + }); + const base = { + id: ids.derive.approvalTurnItem({ requestId }), + threadId: running.input.threadId, + runId: running.input.runId, + nodeId, + providerThreadId: thread.id, + providerTurnId: running.turn.id, + nativeItemRef: nativeRef(native.id), + parentItemId: null, + ordinal: ordinal(native.id), + status: "waiting" as const, + startedAt: at, + completedAt: null, + updatedAt: at, + requestId, + }; + const turnItem: OrchestrationV2TurnItem = question + ? { + ...base, + type: "user_input_request", + title: "Kilo question", + questions: native.questions.map((q, index) => ({ + id: String(index), + header: q.header, + question: q.question, + options: q.options, + multiSelect: q.multiple ?? false, + allowCustomAnswer: q.custom ?? true, + })), + } + : { + ...base, + type: "approval_request", + title: native.permission, + requestKind: kind === "user_input" ? "command" : kind, + prompt: native.patterns.join("\n"), + }; + yield* emit({ type: "turn_item.updated", driver: KILO_PROVIDER, turnItem }); + }); + const handle = Effect.fn("KiloAdapterV2.event")(function* (event: Event) { + const eventSession = + event.type === "message.updated" + ? event.properties.info.sessionID + : event.type === "message.part.updated" + ? event.properties.part.sessionID + : "sessionID" in event.properties + ? event.properties.sessionID + : undefined; + // Child approvals are answered against the asking native session. Child completion is + // projected from the owning task, so it cannot accidentally complete the parent's turn. + if ( + eventSession && + eventSession !== ref?.sessionId && + event.type !== "permission.asked" && + event.type !== "question.asked" + ) + return; + switch (event.type) { + case "message.updated": + roles.set(event.properties.info.id, event.properties.info.role); + timestamps.set( + event.properties.info.id, + DateTime.makeUnsafe(event.properties.info.time.created), + ); + if (event.properties.info.role === "assistant") { + parents.set(event.properties.info.id, event.properties.info.parentID); + if (event.properties.info.time.completed !== undefined) + completedMessages.add(event.properties.info.id); + } + if (active?.messageID === event.properties.info.id) active.admitted = true; + if (event.properties.info.role === "assistant") + yield* messageFromParts( + event.properties.info.id, + event.properties.info.time.completed !== undefined, + ); + return; + case "message.part.updated": + putPart(event.properties.part); + if (event.properties.part.type === "tool") yield* tool(event.properties.part); + else { + if ( + event.properties.part.type === "text" || + event.properties.part.type === "reasoning" + ) + yield* textPart(event.properties.part); + yield* messageFromParts(event.properties.part.messageID); + } + return; + case "message.part.delta": { + const part = parts.get(event.properties.partID); + if ( + part && + (part.type === "text" || part.type === "reasoning") && + event.properties.field === "text" + ) { + const updated = { ...part, text: part.text + event.properties.delta }; + putPart(updated); + yield* textPart(updated); + yield* messageFromParts(part.messageID); + } + return; + } + case "permission.asked": + case "question.asked": + yield* ask(event.properties); + return; + case "session.idle": + if (active?.admitted) yield* reconcile(); + return; + case "session.error": + // Context overflow also emits this event while native compaction continues. + // Only idle + persisted assistant state or process exit proves terminality. + if (active) { + active.reportedError = true; + yield* reconcile(); + } + return; + } + }); + const hydrate = Effect.fn("KiloAdapterV2.hydrate")(function* () { + const currentRef = yield* current(); + const nativeSession = yield* wire(client.read(currentRef)); + const all = yield* wire(client.history(currentRef)); + const revertedAt = nativeSession.revert?.messageID; + const boundary = revertedAt ? all.findIndex((entry) => entry.info.id === revertedAt) : -1; + const history = boundary < 0 ? all : all.slice(0, boundary); + const previousMessages = new Map(messages); + messages.clear(); + parts.clear(); + partsByMessage.clear(); + roles.clear(); + parents.clear(); + completedMessages.clear(); + for (const entry of history) { + if (entry.info.role === "assistant") { + parents.set(entry.info.id, entry.info.parentID); + if (entry.info.time.completed) completedMessages.add(entry.info.id); + } + timestamps.set(entry.info.id, DateTime.makeUnsafe(entry.info.time.created)); + const previous = previousMessages.get(entry.info.id); + if (previous) messages.set(entry.info.id, previous); + roles.set(entry.info.id, entry.info.role); + if (entry.info.id === active?.messageID) active.admitted = true; + for (const part of entry.parts) putPart(part); + // Snapshot reads must not re-publish historical records over T3's durable run metadata. + yield* messageFromParts( + entry.info.id, + true, + active !== undefined && + (entry.info.id === active.messageID || + parents.get(entry.info.id) === active.messageID), + ); + } + const users = history.filter((entry) => entry.info.role === "user"); + const correlations = new Map( + Object.entries(thread?.nativeMetadata?.turnCorrelations ?? {}).map(([id, value]) => [ + resolveMessage(id), + { id, value }, + ]), + ); + for (const [index, entry] of users.entries()) { + const correlation = correlations.get(entry.info.id); + const nativeId = correlation?.id ?? entry.info.id; + const id = ids.derive.providerTurn({ + driver: KILO_PROVIDER, + nativeTurnId: itemKey(nativeId), + }); + if (providerTurns.has(id) || !thread) continue; + const replies = history.filter( + (m) => m.info.role === "assistant" && m.info.parentID === entry.info.id, + ); + const last = replies.at(-1)?.info; + const failed = last?.role === "assistant" ? last.error : undefined; + const ended = last?.role === "assistant" ? last.time.completed : undefined; + providerTurns.set(id, { + id, + providerThreadId: thread.id, + nodeId: + correlation?.value.nodeId ?? + ids.derive.nodeFromProviderItem({ + driver: KILO_PROVIDER, + nativeItemId: itemKey(nativeId), + }), + runAttemptId: correlation?.value.attemptId ?? null, + nativeTurnRef: nativeRef(entry.info.id), + ordinal: correlation?.value.ordinal ?? index + 1, + status: + correlation?.value.terminalStatus ?? + (failed + ? failed.name === "MessageAbortedError" + ? "interrupted" + : "failed" + : ended + ? "completed" + : "interrupted"), + startedAt: DateTime.makeUnsafe(entry.info.time.created), + completedAt: correlation?.value.completedAt + ? DateTime.makeUnsafe(correlation.value.completedAt) + : ended + ? DateTime.makeUnsafe(ended) + : null, + }); + } + return history; + }); + const reconcile = Effect.fn("KiloAdapterV2.reconcile")(function* () { + const native = yield* current(); + const history = yield* hydrate(); + if (!active) return; + const pending = yield* wire(client.pending(native, true)); + const present = new Set(pending.map((request) => request.id)); + for (const request of pending) yield* ask(request); + for (const saved of requests.values()) { + if (saved.runtime.status !== "pending" || present.has(saved.native.id)) continue; + const at = yield* DateTime.now; + saved.runtime = { ...saved.runtime, status: "cancelled", resolvedAt: at }; + yield* emit({ + type: "runtime_request.updated", + driver: KILO_PROVIDER, + runtimeRequest: saved.runtime, + }); + const node = nodes.get(saved.runtime.nodeId); + if (node) + yield* emit({ + type: "node.updated", + driver: KILO_PROVIDER, + node: { ...node, status: "interrupted", completedAt: at }, + }); + const item = items.get(ids.derive.approvalTurnItem({ requestId: saved.runtime.id })); + if (item) + yield* emit({ + type: "turn_item.updated", + driver: KILO_PROVIDER, + turnItem: { ...item, status: "interrupted", completedAt: at, updatedAt: at }, + }); + } + if (!active?.admitted) return; + const status = yield* wire(client.status(native)); + if (status !== "idle") return; + const replies = history.filter( + (m) => m.info.role === "assistant" && m.info.parentID === active?.messageID, + ); + const last = replies.at(-1)?.info; + if (active.interrupting) return; + // A successful reply after compaction supersedes the earlier recoverable error. + if (last?.role === "assistant" && last.time.completed !== undefined && !last.error) + yield* finish("completed"); + else if ((last?.role === "assistant" && last.error) || active.reportedError) + yield* finish("failed", "Kilo recorded an error while processing this turn."); + }); + let subscribed = false; + let eventFiber: Fiber.Fiber | undefined; + const startEvents = Effect.fn("KiloAdapterV2.subscribe")(function* () { + if (subscribed) return; + const ready = yield* Deferred.make(); + const native = yield* current(); + const watch = Effect.gen(function* () { + while (yield* connection.isRunning) { + yield* client + .events( + native, + Deferred.succeed(ready, undefined).pipe( + Effect.andThen( + Effect.suspend(() => + active + ? reconcile().pipe( + Effect.catchDefect(() => + Effect.fail(error("Kilo returned invalid recovery data")), + ), + Effect.mapError( + (cause) => + new KiloSessionError({ + operation: "reconnect", + reason: "request_failed", + cause, + }), + ), + ) + : Effect.void, + ), + ), + ), + true, + ) + .pipe( + Stream.runForEach((event) => + handle(event).pipe( + Effect.catchDefect(() => + Effect.fail(error("Kilo emitted an invalid event payload")), + ), + ), + ), + Effect.catch(() => + Effect.gen(function* () { + yield* Deferred.fail(ready, error("Kilo stream did not become ready")); + if (!(yield* connection.isRunning)) { + if (active && !active.interrupting) { + yield* connection.cleanup; + yield* finish("failed", "Kilo process exited."); + } + return; + } + yield* reconcile().pipe( + Effect.catchDefect(() => Effect.void), + Effect.ignore, + ); + yield* emit({ + type: "provider_session.updated", + driver: KILO_PROVIDER, + providerSession: { + ...session, + status: "error", + lastError: + "Kilo stream disconnected; reconnecting. The task may still be running.", + updatedAt: yield* DateTime.now, + }, + }); + yield* Effect.sleep("1 second"); + }), + ), + ); + } + }); + eventFiber = yield* watch.pipe(Effect.forkIn(scope)); + yield* Deferred.await(ready).pipe( + Effect.timeout("10 seconds"), + Effect.mapError(() => error("Kilo stream readiness timed out")), + ); + subscribed = true; + }); + const bind = Effect.fn("KiloAdapterV2.bind")(function* ( + native: KiloSessionRef, + appThreadId: OrchestrationV2ProviderThread["appThreadId"], + existing?: OrchestrationV2ProviderThread, + ) { + if (ref && ref.sessionId !== native.sessionId && subscribed) + return yield* error("This Kilo session already owns another native thread"); + ref = native; + thread = existing + ? { + ...existing, + providerSessionId: input.providerSessionId, + status: "idle", + nativeThreadRef: nativeRef(native.sessionId), + nativeMetadata: existing.nativeThreadRef + ? existing.nativeMetadata + : { continuationKey: options.continuationKey, itemIdentityVersion: 2 }, + } + : { + id: ids.derive.providerThread({ + driver: KILO_PROVIDER, + providerInstanceId: options.instanceId, + nativeThreadId: itemKey(native.sessionId), + }), + driver: KILO_PROVIDER, + providerInstanceId: options.instanceId, + providerSessionId: input.providerSessionId, + appThreadId, + ownerNodeId: null, + nativeThreadRef: nativeRef(native.sessionId), + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + nativeMetadata: { continuationKey: options.continuationKey, itemIdentityVersion: 2 }, + createdAt: now, + updatedAt: now, + }; + yield* startEvents(); + return thread; + }); + const snapshot = Effect.fn("KiloAdapterV2.snapshot")(function* () { + yield* hydrate(); + if (!thread) return yield* error("Kilo thread is not loaded"); + return { + providerThread: thread, + providerTurns: [...providerTurns.values()], + messages: [...messages.values()], + runtimeRequests: [...requests.values()].map((r) => r.runtime), + }; + }); + const resumeThread = ( + request: Parameters[0], + ) => + Effect.gen(function* () { + if (active) return yield* error("Stop the Kilo task before restoring a conversation"); + const saved = request.providerThread; + if ( + saved.providerInstanceId !== options.instanceId || + saved.nativeMetadata?.continuationKey !== options.continuationKey || + !saved.nativeThreadRef?.nativeId + ) + return yield* error( + "Kilo account or configuration changed; the previous native session cannot be resumed", + ); + const native = { + instanceId: options.continuationKey, + directory, + sessionId: saved.nativeThreadRef.nativeId, + }; + // Runtime's client uses the same account-scoped instance identity. + yield* wire(client.read(native)); + yield* wire( + client.setPermissions( + native, + permissions(request.runtimePolicy ?? input.runtimePolicy), + ), + ); + return yield* bind(native, saved.appThreadId, saved); + }); + const runtime: Adapter.ProviderAdapterV2SessionRuntime = { + instanceId: options.instanceId, + driver: KILO_PROVIDER, + providerSessionId: input.providerSessionId, + providerSession: session, + events: Stream.fromEffectRepeat(Queue.take(events)), + ensureThread: (request) => + turnGate.withPermit( + Effect.gen(function* () { + const prior = request.existingProviderThread; + if ( + prior?.nativeMetadata?.continuationKey && + prior.nativeMetadata.continuationKey !== options.continuationKey + ) + return yield* error("Kilo account changed; start a new thread for this account"); + if (prior?.nativeThreadRef) return yield* resumeThread({ providerThread: prior }); + if (active) + return yield* error("Stop the Kilo task before replacing its conversation"); + const native = yield* wire(client.create(permissions(request.runtimePolicy))); + return yield* bind(native, request.threadId, prior); + }), + ), + resumeThread: (request) => turnGate.withPermit(resumeThread(request)), + startTurn: (request) => + turnGate.withPermit( + Effect.gen(function* () { + const native = yield* ownedThread(request.providerThread); + if ( + request.threadId !== thread?.appThreadId || + request.appThread.id !== request.threadId || + (request.runtimePolicy.cwd ?? options.cwd) !== directory + ) + return yield* error("Kilo turn does not belong to this workspace and thread"); + if (!(yield* connection.isRunning)) + return yield* error( + "Kilo process stopped; resume the session before starting a turn", + ); + if (active) + return yield* error("A Kilo task is still running or its admission is unresolved"); + if (request.message.attachments.length && !options.attachmentsDir) + return yield* error("Kilo attachment storage is unavailable"); + const files = toOpenCodeFileParts({ + attachments: request.message.attachments, + resolveAttachmentPath: (attachment) => + options.attachmentsDir + ? resolveAttachmentPath({ attachmentsDir: options.attachmentsDir, attachment }) + : null, + }); + yield* wire(client.setPermissions(native, permissions(request.runtimePolicy))); + const slash = request.modelSelection.model.indexOf("/"); + if (slash <= 0) return yield* error("Kilo models must use provider/model format"); + const startedAt = yield* DateTime.now; + const uuid = yield* crypto.randomUUIDv4.pipe( + Effect.mapError(() => error("Could not allocate a Kilo message identity")), + ); + const messageID = `msg_${(BigInt(DateTime.toEpochMillis(startedAt)) * 4096n).toString(16)}${uuid.replaceAll("-", "")}`; + const turn: OrchestrationV2ProviderTurn = { + id: ids.derive.providerTurn({ + driver: KILO_PROVIDER, + nativeTurnId: itemKey(messageID), + }), + providerThreadId: request.providerThread.id, + nodeId: request.rootNodeId, + runAttemptId: request.attemptId, + nativeTurnRef: nativeRef(messageID), + ordinal: request.providerTurnOrdinal, + status: "running", + startedAt, + completedAt: null, + }; + active = { input: request, turn, messageID, admitted: false, interrupting: false }; + thread = { + ...request.providerThread, + nativeMetadata: { + ...thread?.nativeMetadata, + turnCorrelations: { + ...thread?.nativeMetadata?.turnCorrelations, + [messageID]: { + messageId: request.message.messageId, + nodeId: request.rootNodeId, + runId: request.runId, + attemptId: request.attemptId, + ordinal: request.providerTurnOrdinal, + attachments: request.message.attachments, + createdBy: request.message.createdBy, + creationSource: request.message.creationSource, + scheduledTaskId: request.message.scheduledTaskId, + senderThreadId: request.message.senderThreadId, + }, + }, + }, + }; + yield* emit({ + type: "provider_thread.updated", + driver: KILO_PROVIDER, + providerThread: thread, + }); + ordinals.clear(); + nodes.clear(); + items.clear(); + parts.clear(); + partsByMessage.clear(); + parents.clear(); + roles.clear(); + providerTurns.set(turn.id, turn); + yield* emit({ + type: "provider_turn.updated", + driver: KILO_PROVIDER, + providerTurn: turn, + }); + yield* client + .prompt(native, { + messageID, + model: { + providerID: request.modelSelection.model.slice(0, slash), + modelID: request.modelSelection.model.slice(slash + 1), + }, + agent: + request.runtimePolicy.interactionMode === "plan" + ? "plan" + : (getModelSelectionStringOptionValue(request.modelSelection, "agent") ?? + "build"), + ...(getModelSelectionStringOptionValue(request.modelSelection, "variant") + ? { + variant: getModelSelectionStringOptionValue( + request.modelSelection, + "variant", + )!, + } + : {}), + parts: [{ type: "text", text: request.message.text }, ...files], + }) + .pipe( + Effect.tap(() => + Effect.gen(function* () { + if (active?.turn.id !== turn.id) return; + active.admitted = true; + if (active.reportedError) yield* reconcile().pipe(Effect.ignore); + }), + ), + Effect.catch((cause) => + Effect.gen(function* () { + // A timeout after admission must not fail startTurn: T3 would detach its event consumer. + if (cause.reason !== "admission_unknown") { + yield* finish("failed", "Kilo rejected the prompt."); + return; + } + yield* reconcile().pipe( + Effect.catchDefect(() => Effect.void), + Effect.ignore, + ); + if (!active) return; + yield* emit({ + type: "provider_session.updated", + driver: KILO_PROVIDER, + providerSession: { + ...session, + status: "waiting", + lastError: + cause.reason === "admission_unknown" + ? "Prompt acceptance is uncertain. Kilo is still being monitored; do not resubmit. Stop the session to cancel safely." + : "Kilo did not confirm prompt acceptance. Stop this session before retrying.", + updatedAt: yield* DateTime.now, + }, + }); + }), + ), + ); + }), + ), + steerTurn: (request) => + Effect.fail( + new Adapter.ProviderAdapterSteerRunUnsupportedError({ + driver: KILO_PROVIDER, + providerThreadId: request.providerThread.id, + }), + ), + interruptTurn: (request) => + turnGate.withPermit( + Effect.gen(function* () { + const native = yield* ownedThread(request.providerThread); + // A delayed Stop for the previous turn must not terminate its successor. + if (!active || active.turn.id !== request.providerTurnId) return; + active.interrupting = true; + yield* client.abort(native).pipe(Effect.ignore); + // Killing this session's owned process also covers uncertain admission and descendant tools. + yield* connection.stop; + yield* finish("interrupted"); + }), + ), + respondToRuntimeRequest: (request) => + turnGate.withPermit( + Effect.gen(function* () { + const pending = requests.get(request.requestId); + if (!pending || pending.runtime.status !== "pending") + return yield* error("Kilo request is no longer pending"); + const root = yield* current(); + const native = { ...root, sessionId: pending.native.sessionID }; + // Requests enter this map only after the stream verifies the full parent chain. + // The client rechecks both the session owner and pending request before replying. + if ("questions" in pending.native) { + if (!request.answers) return yield* error("Kilo question requires answers"); + const answers = pending.native.questions.map((_, index) => { + const answer = request.answers?.[String(index)]; + return typeof answer === "string" + ? [answer] + : Array.isArray(answer) && answer.every((value) => typeof value === "string") + ? answer + : []; + }); + if (answers.some((answer) => answer.length === 0)) + return yield* error("Each Kilo question requires a text answer"); + yield* wire(client.replyQuestion(native, pending.native.id, answers)); + } else { + if (!request.decision) return yield* error("Kilo approval requires a decision"); + yield* wire( + client.replyPermission( + native, + pending.native.id, + request.decision === "accept" + ? "once" + : request.decision === "acceptForSession" || + request.decision === "acceptAlways" + ? "always" + : "reject", + ), + ); + } + const resolved = { + ...pending.runtime, + status: "resolved" as const, + resolvedAt: yield* DateTime.now, + }; + pending.runtime = resolved; + yield* emit({ + type: "runtime_request.updated", + driver: KILO_PROVIDER, + runtimeRequest: resolved, + }); + const node = nodes.get(pending.runtime.nodeId); + if (node) + yield* emit({ + type: "node.updated", + driver: KILO_PROVIDER, + node: { ...node, status: "completed", completedAt: resolved.resolvedAt }, + }); + const item = items.get(ids.derive.approvalTurnItem({ requestId: request.requestId })); + if (item) + yield* emit({ + type: "turn_item.updated", + driver: KILO_PROVIDER, + turnItem: { + ...item, + status: "completed", + completedAt: resolved.resolvedAt, + updatedAt: resolved.resolvedAt, + }, + }); + }), + ), + readThreadSnapshot: (request) => + turnGate.withPermit( + Effect.gen(function* () { + yield* ownedThread(request.providerThread); + if (active) return yield* error("Stop the Kilo task before reading its history"); + return yield* snapshot(); + }), + ), + rollbackThread: (request) => + turnGate.withPermit( + Effect.gen(function* () { + const native = yield* ownedThread(request.providerThread); + if (active) return yield* error("Stop the Kilo task before rewinding"); + const history = yield* wire(client.history(native)); + const target = + request.target.type === "provider_turn" ? request.target.providerTurn : undefined; + const targetId = target?.nativeTurnRef?.nativeId; + if ( + target && + (!targetId || !history.some((m) => m.info.id === resolveMessage(targetId))) + ) + return yield* error("Kilo rewind boundary is unavailable"); + const targetIndex = targetId + ? history.findIndex((m) => m.info.id === resolveMessage(targetId)) + : -1; + const next = history.slice(targetIndex + 1).find((m) => m.info.role === "user") + ?.info.id; + if (!next) return yield* snapshot(); + // Native revert changes files. Fork only the conversation; T3 owns filesystem rewind. + const fork = yield* wire(client.fork(native, next)); + const retained = history.slice( + 0, + history.findIndex((m) => m.info.id === next), + ); + const copied = yield* wire(client.history(fork)); + if ( + copied.length !== retained.length || + copied.some((m, i) => m.info.role !== retained[i]?.info.role) + ) + return yield* error("Kilo fork returned an unexpected conversation boundary"); + const aliases = { ...thread?.nativeMetadata?.messageAliases }; + for (const [i, entry] of retained.entries()) { + const replacement = copied[i]!.info.id; + for (const [old, currentId] of Object.entries(aliases)) + if (currentId === entry.info.id) aliases[old] = replacement; + aliases[entry.info.id] = replacement; + } + if (eventFiber) yield* Fiber.interrupt(eventFiber); + subscribed = false; + ref = fork; + thread = { + ...thread!, + nativeThreadRef: nativeRef(fork.sessionId), + nativeMetadata: { + ...thread?.nativeMetadata, + messageAliases: aliases, + }, + }; + providerTurns.clear(); + for (const turn of request.providerThreadTurns) { + if (target && turn.ordinal <= target.ordinal) + providerTurns.set(turn.id, { + ...turn, + nativeTurnRef: turn.nativeTurnRef?.nativeId + ? nativeRef(resolveMessage(turn.nativeTurnRef.nativeId)) + : turn.nativeTurnRef, + }); + } + requests.clear(); + yield* startEvents(); + return yield* snapshot(); + }), + ), + forkThread: (request) => + turnGate.withPermit( + Effect.gen(function* () { + const native = yield* ownedThread(request.sourceProviderThread); + if (active) return yield* error("Stop the Kilo task before forking"); + const boundary = request.providerTurnId + ? request.sourceProviderTurns?.find((turn) => turn.id === request.providerTurnId) + : undefined; + if (request.providerTurnId && !boundary) + return yield* error("Kilo fork boundary is unavailable"); + const history = yield* wire(client.history(native)); + const boundaryId = boundary?.nativeTurnRef?.nativeId; + const boundaryIndex = boundaryId + ? history.findIndex((m) => m.info.id === resolveMessage(boundaryId)) + : -1; + if (boundary && boundaryIndex < 0) + return yield* error("Kilo fork boundary no longer exists"); + const next = boundary + ? history.slice(boundaryIndex + 1).find((m) => m.info.role === "user")?.info.id + : undefined; + const fork = yield* wire(client.fork(native, next)); + const retained = next + ? history.slice( + 0, + history.findIndex((m) => m.info.id === next), + ) + : history; + const copied = yield* wire(client.history(fork)); + if ( + copied.length !== retained.length || + copied.some((m, i) => m.info.role !== retained[i]?.info.role) + ) + return yield* error("Kilo fork returned an unexpected conversation boundary"); + const aliases = { ...thread?.nativeMetadata?.messageAliases }; + for (const [i, entry] of retained.entries()) { + for (const [old, currentId] of Object.entries(aliases)) + if (currentId === entry.info.id) aliases[old] = copied[i]!.info.id; + aliases[entry.info.id] = copied[i]!.info.id; + } + if (!thread) return yield* error("Kilo thread is not loaded"); + return { + ...thread, + id: ids.derive.providerThread({ + driver: KILO_PROVIDER, + providerInstanceId: options.instanceId, + nativeThreadId: itemKey(fork.sessionId), + }), + nativeThreadRef: nativeRef(fork.sessionId), + appThreadId: request.targetThreadId, + nativeMetadata: { + ...thread.nativeMetadata, + messageAliases: aliases, + turnCorrelations: {}, + }, + providerSessionId: null, + forkedFrom: { + providerThreadId: request.sourceProviderThread.id, + ...(request.providerTurnId ? { providerTurnId: request.providerTurnId } : {}), + }, + }; + }), + ), + }; + return runtime; + }), + }); +}); diff --git a/apps/server/src/provider/Drivers/KiloDriver.ts b/apps/server/src/provider/Drivers/KiloDriver.ts new file mode 100644 index 000000000000..1772910f50e4 --- /dev/null +++ b/apps/server/src/provider/Drivers/KiloDriver.ts @@ -0,0 +1,270 @@ +import { KiloSettings, type ServerProvider } from "@t3tools/contracts"; +import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Encoding from "effect/Encoding"; +import * as Path from "effect/Path"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import type * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import * as KiloTextGeneration from "../../textGeneration/KiloTextGeneration.ts"; +import * as ServerConfig from "../../config.ts"; +import * as KiloAdapter from "../../orchestration-v2/Adapters/KiloAdapterV2.ts"; +import * as IdAllocator from "../../orchestration-v2/IdAllocator.ts"; +import * as KiloRuntime from "../kilo/KiloRuntime.ts"; +import { ProviderDriverError } from "../Errors.ts"; +import type { ProviderDriver } from "../ProviderDriver.ts"; +import { buildServerProvider } from "../providerSnapshot.ts"; +import { makeManualOnlyProviderMaintenanceCapabilities } from "../providerMaintenance.ts"; + +const decode = Schema.decodeSync(KiloSettings); +const kind = KiloAdapter.KILO_PROVIDER; +const systemKeys = new Set([ + "PATH", + "HOME", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "SYSTEMROOT", + "SystemRoot", + "WINDIR", + "COMSPEC", + "ComSpec", + "PATHEXT", + "TMP", + "TEMP", + "TMPDIR", + "LANG", + "LC_ALL", + "TERM", + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "no_proxy", + "NODE_EXTRA_CA_CERTS", + "SSL_CERT_FILE", + "SSL_CERT_DIR", +]); + +export type KiloDriverEnv = + | Exclude>, Scope.Scope> + | IdAllocator.IdAllocatorV2 + | ServerConfig.ServerConfig; + +export const KiloDriver: ProviderDriver = { + driverKind: kind, + metadata: { displayName: "Kilo", supportsMultipleInstances: true }, + configSchema: KiloSettings, + defaultConfig: () => decode({}), + create: Effect.fn("KiloDriver.create")(function* (input) { + const server = yield* ServerConfig.ServerConfig; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const host = yield* HostProcessEnvironment; + // Only explicit per-instance variables may select provider credentials/config. + const environment: NodeJS.ProcessEnv = Object.fromEntries( + Object.entries(host).filter(([key]) => systemKeys.has(key)), + ); + for (const variable of input.environment) environment[variable.name] = variable.value; + const profileDirectory = + input.config.profileDirectory || + path.join( + server.stateDir, + "providers", + "kilo", + input.instanceId, + encodeURIComponent(input.config.accountId), + ); + const identity = [ + path.resolve(profileDirectory), + input.config.accountId, + ...input.environment + .toSorted((a, b) => a.name.localeCompare(b.name)) + .map((entry) => `${entry.name}=${entry.value}`), + ].join("\0"); + const digest = yield* crypto.digest("SHA-256", new TextEncoder().encode(identity)).pipe( + Effect.mapError( + (cause) => + new ProviderDriverError({ + driver: kind, + instanceId: input.instanceId, + detail: "Could not establish Kilo account identity.", + cause, + }), + ), + ); + const continuationKey = `kilo:${input.instanceId}:${Encoding.encodeHex(digest)}`; + const runtime = yield* KiloRuntime.make({ + instanceId: continuationKey, + binaryPath: input.config.binaryPath, + profileDirectory, + environment, + }); + const orchestrationAdapter = yield* KiloAdapter.make({ + instanceId: input.instanceId, + continuationKey, + cwd: server.cwd, + attachmentsDir: server.attachmentsDir, + runtime, + }); + const changes = yield* PubSub.unbounded(); + const mutex = yield* Semaphore.make(1); + const stamp = (snapshot: ReturnType): ServerProvider => ({ + ...snapshot, + instanceId: input.instanceId, + driver: kind, + displayName: input.displayName ?? "Kilo", + ...(input.accentColor ? { accentColor: input.accentColor } : {}), + continuation: { groupKey: continuationKey }, + }); + let latest = stamp( + buildServerProvider({ + driver: kind, + presentation: { + displayName: "Kilo", + badgeLabel: "Preview", + supportsConversationRollback: true, + supportedRuntimeModes: ["full-access", "approval-required"], + }, + enabled: input.enabled, + checkedAt: DateTime.formatIso(yield* DateTime.now), + models: [], + probe: { + installed: false, + version: null, + status: "warning", + auth: { status: "unknown", profileId: input.config.accountId }, + message: "Kilo has not been checked yet.", + }, + }), + ); + const refresh = mutex.withPermit( + Effect.gen(function* () { + if (!input.enabled) return latest; + const check = Effect.gen(function* () { + const connection = yield* runtime.open(server.cwd); + const inventory = yield* connection.client.models(); + const agents = (yield* connection.client.agents()).filter( + (agent) => !agent.hidden && (agent.mode === "primary" || agent.mode === "all"), + ); + const models = inventory.all + .filter((provider) => inventory.connected.includes(provider.id)) + .flatMap((provider) => + Object.values(provider.models).map((model) => ({ + slug: `${provider.id}/${model.id}`, + name: model.name, + subProvider: provider.name, + isCustom: false, + capabilities: { + optionDescriptors: [ + ...(Object.keys(model.variants ?? {}).length + ? [ + { + id: "variant", + label: "Reasoning", + type: "select" as const, + options: Object.keys(model.variants ?? {}).map((variant) => ({ + id: variant, + label: variant, + })), + }, + ] + : []), + ...(agents.length + ? [ + { + id: "agent", + label: "Agent", + type: "select" as const, + options: agents.map((agent) => ({ + id: agent.name, + label: agent.displayName ?? agent.name, + ...(agent.name === "build" ? { isDefault: true } : {}), + })), + }, + ] + : []), + ], + }, + })), + ); + return stamp( + buildServerProvider({ + driver: kind, + presentation: { + displayName: "Kilo", + badgeLabel: "Preview", + supportsConversationRollback: true, + supportedRuntimeModes: ["full-access", "approval-required"], + }, + enabled: input.enabled, + checkedAt: DateTime.formatIso(yield* DateTime.now), + models, + probe: { + installed: true, + version: connection.client.version, + status: models.length ? "ready" : "warning", + auth: { + status: "unknown", + profileId: input.config.accountId, + }, + ...(models.length + ? { + message: + "Kilo is ready. Model availability and authentication have not been verified by running a prompt. Subagents require Full access mode; Kilo-configured approvals still apply.", + } + : { + message: `No connected models in the Kilo account profile ${profileDirectory}. Configure this profile with Kilo or set explicit provider environment variables.`, + }), + }, + }), + ); + }).pipe(Effect.scoped); + latest = yield* check.pipe( + Effect.catch(() => + Effect.succeed({ + ...latest, + status: "error" as const, + installed: false, + message: + "Kilo could not be checked. Verify the binary, profile and CLI version 7.8.3.", + }), + ), + ); + yield* PubSub.publish(changes, latest); + return latest; + }), + ); + if (input.enabled) yield* refresh.pipe(Effect.forkScoped); + const textGeneration = KiloTextGeneration.make(runtime, server.attachmentsDir); + return { + instanceId: input.instanceId, + driverKind: kind, + displayName: input.displayName, + accentColor: input.accentColor, + enabled: input.enabled, + continuationIdentity: { driverKind: kind, continuationKey }, + orchestrationAdapter, + textGeneration, + snapshot: { + getSnapshot: Effect.sync(() => latest), + refresh, + streamChanges: Stream.fromPubSub(changes), + applyUsageLimits: () => Effect.void, + resolveMaintenance: () => + Effect.succeed( + makeManualOnlyProviderMaintenanceCapabilities({ + provider: kind, + packageName: "@kilocode/cli", + }), + ), + }, + }; + }), +}; diff --git a/apps/server/src/provider/builtInDrivers.ts b/apps/server/src/provider/builtInDrivers.ts index 1e6d7beb61ea..c82cfd564322 100644 --- a/apps/server/src/provider/builtInDrivers.ts +++ b/apps/server/src/provider/builtInDrivers.ts @@ -27,6 +27,7 @@ import { CodexDriver, type CodexDriverEnv } from "./Drivers/CodexDriver.ts"; import { CursorDriver, type CursorDriverEnv } from "./Drivers/CursorDriver.ts"; import { GrokDriver, type GrokDriverEnv } from "./Drivers/GrokDriver.ts"; import { OpenCodeDriver, type OpenCodeDriverEnv } from "./Drivers/OpenCodeDriver.ts"; +import { KiloDriver, type KiloDriverEnv } from "./Drivers/KiloDriver.ts"; import { PiDriver, type PiDriverEnv } from "./Drivers/PiDriver.ts"; import type { AnyProviderDriver } from "./ProviderDriver.ts"; @@ -43,7 +44,8 @@ export type BuiltInDriversEnv = | CursorDriverEnv | GrokDriverEnv | OpenCodeDriverEnv - | PiDriverEnv; + | PiDriverEnv + | KiloDriverEnv; /** * Ordered list of built-in drivers. Order matters only for tie-breaking in @@ -56,6 +58,7 @@ export const BUILT_IN_DRIVERS: ReadonlyArray Promise> = []; +afterEach(async () => { + for (const cleanup of cleanups.splice(0)) await cleanup(); +}); +const sessionId = "agent_12345678-1234-1234-1234-123456789abc"; +const messageId = "msg_0123456789ab0123456789ABCD"; +const ref: Cloud.KiloCloudRef = { accountKey: "account-a", sessionId, messageId }; +const start = { + messageId, + prompt: "Contract fixture only", + repository: { type: "github" as const, repo: "fixture/project" }, + model: "fixture/model", + mode: "code", +}; +const run = Effect.runPromise; +async function client( + handler: (request: NodeHttp.IncomingMessage, response: NodeHttp.ServerResponse) => void, +) { + const server = NodeHttp.createServer(handler); + server.listen(0, "127.0.0.1"); + await NodeEvents.EventEmitter.once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing fixture address"); + cleanups.push(async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }); + return Cloud.make({ + accountKey: ref.accountKey, + apiKey: Redacted.make("fixture-customer-token"), + origin: `http://127.0.0.1:${address.port}`, + }); +} +function json(response: NodeHttp.ServerResponse, data: unknown) { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ result: { data } })); +} +describe("Kilo customer Cloud Agent boundary", () => { + it("sends one bearer-authenticated admission and validates its caller-persisted message ID", async () => { + const requests: unknown[] = []; + const cloud = await client((request, response) => { + expect(request.headers.authorization).toBe("Bearer fixture-customer-token"); + expect(request.url).toBe("/trpc/start"); + let body = ""; + request.on("data", (chunk) => { + body += String(chunk); + }); + request.on("end", () => { + requests.push(JSON.parse(body)); + json(response, { + cloudAgentSessionId: sessionId, + kiloSessionId: "ses_remote", + messageId, + delivery: "queued", + }); + }); + }); + expect(await run(cloud.start(start))).toEqual(ref); + expect(requests).toEqual([ + { + message: { id: messageId, prompt: start.prompt }, + repository: start.repository, + agent: { model: start.model, mode: start.mode }, + options: { createdOnPlatform: "kilo-cli" }, + }, + ]); + }); + it("does not retry or declare stopped after a response is lost following admission", async () => { + let accepted = 0; + const cloud = await client((request, response) => { + request.resume(); + request.on("end", () => { + accepted++; + response.destroy(); + }); + }); + const failure = await run(cloud.start(start).pipe(Effect.flip)); + expect(failure.reason).toBe("admission_unknown"); + expect(failure.messageId).toBe(messageId); + expect(accepted).toBe(1); + }); + it.each([500, 503, 307])("never follows or retries mutation status %i", async (status) => { + let requests = 0; + const cloud = await client((_request, response) => { + requests++; + response.writeHead(status, { location: "/other" }); + response.end(); + }); + expect((await run(cloud.start(start).pipe(Effect.flip))).reason).toBe("admission_unknown"); + expect(requests).toBe(1); + }); + it("rejects a mismatched session on send and a mismatched message on result", async () => { + const cloud = await client((request, response) => + json( + response, + request.method === "POST" + ? { + cloudAgentSessionId: "agent_aaaaaaaa-1234-1234-1234-123456789abc", + messageId, + delivery: "started", + } + : { + cloudAgentSessionId: sessionId, + messageId: "msg_aaaaaaaaaaaa0123456789ABCD", + status: "completed", + createdAt: 1, + }, + ), + ); + expect((await run(cloud.send(ref, messageId, "test").pipe(Effect.flip))).reason).toBe( + "admission_unknown", + ); + expect((await run(cloud.result(ref).pipe(Effect.flip))).reason).toBe("wrong_owner"); + }); + it("cannot reuse another account's ref or issue an invented stop request", async () => { + let requests = 0; + const cloud = await client((_request, response) => { + requests++; + response.end(); + }); + expect( + (await run(cloud.result({ ...ref, accountKey: "account-b" }).pipe(Effect.flip))).reason, + ).toBe("wrong_owner"); + expect((await run(cloud.interrupt(ref).pipe(Effect.flip))).reason).toBe("unsupported"); + expect(requests).toBe(0); + }); + it("keeps billing unknown even when a task has a confirmed terminal result", async () => { + const cloud = await client((_request, response) => + json(response, { + cloudAgentSessionId: sessionId, + messageId, + status: "interrupted", + createdAt: 1, + terminalAt: 2, + }), + ); + const result = await run(cloud.result(ref)); + expect(result.status).toBe("interrupted"); + expect(result.billingStatus).toBe("unknown"); + }); +}); diff --git a/apps/server/src/provider/kilo/KiloCloudClient.ts b/apps/server/src/provider/kilo/KiloCloudClient.ts new file mode 100644 index 000000000000..1259c981f353 --- /dev/null +++ b/apps/server/src/provider/kilo/KiloCloudClient.ts @@ -0,0 +1,214 @@ +import * as Effect from "effect/Effect"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; + +/** Customer bearer endpoints used by the public Kilo 7.8.3 CLI, not the private cloud SDK. + * Reference: Kilo-Org/kilo packages/opencode/src/kilocode/cloud/{trpc,contracts}.ts. + * This boundary deliberately cannot report that billing stopped or invent a remote Stop route. + */ +export class KiloCloudError extends Schema.TaggedError()("KiloCloudError", { + operation: Schema.String, + reason: Schema.Literals([ + "rejected", + "admission_unknown", + "invalid_response", + "wrong_owner", + "unsupported", + ]), + messageId: Schema.optional(Schema.String), +}) {} + +export const KiloCloudRef = Schema.Struct({ + accountKey: Schema.NonEmptyString, + sessionId: Schema.String.check(Schema.isPattern(/^agent_[0-9a-f-]{36}$/i)), + messageId: Schema.String.check(Schema.isPattern(/^msg_[0-9a-f]{12}[0-9A-Za-z]{14}$/)), +}); +export type KiloCloudRef = typeof KiloCloudRef.Type; +const Admission = Schema.Struct({ + cloudAgentSessionId: KiloCloudRef.fields.sessionId, + kiloSessionId: Schema.optional(Schema.String), + messageId: KiloCloudRef.fields.messageId, + delivery: Schema.NonEmptyString, +}); +const Result = Schema.Struct({ + cloudAgentSessionId: KiloCloudRef.fields.sessionId, + messageId: KiloCloudRef.fields.messageId, + status: Schema.Literals(["queued", "running", "completed", "failed", "interrupted"]), + createdAt: Schema.Number, + terminalAt: Schema.optional(Schema.Number), + assistant: Schema.optional( + Schema.Struct({ messageId: Schema.String, text: Schema.optional(Schema.String) }), + ), +}); +const Envelope = Schema.Struct({ result: Schema.Struct({ data: Schema.Unknown }) }); +const decodeEnvelope = Schema.decodeUnknownEffect(Schema.fromJsonString(Envelope)); +const decodeAdmission = Schema.decodeUnknownEffect(Admission); +const decodeResult = Schema.decodeUnknownEffect(Result); +const isCloudError = Schema.is(KiloCloudError); +const encode = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); + +export const make = (input: { + readonly accountKey: string; + readonly apiKey: Redacted.Redacted; + /** Only the production customer endpoint or a local contract-test server. */ + readonly origin?: string; +}) => { + const origin = input.origin ?? "https://cloud-agent-next.kilosessions.ai"; + const allowedOrigin = + origin === "https://cloud-agent-next.kilosessions.ai" || + /^http:\/\/127\.0\.0\.1:\d+$/.test(origin); + const own = (ref: KiloCloudRef) => + ref.accountKey === input.accountKey + ? Effect.void + : Effect.fail(new KiloCloudError({ operation: "ownership", reason: "wrong_owner" })); + const request = ( + operation: "start" | "send" | "getMessageResult", + body: unknown, + messageId: string, + ) => { + const mutation = operation !== "getMessageResult"; + const uncertain = () => + new KiloCloudError({ + operation, + reason: mutation ? "admission_unknown" : "invalid_response", + messageId, + }); + if (!allowedOrigin) return Effect.fail(new KiloCloudError({ operation, reason: "rejected" })); + return Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + const req = HttpClientRequest.make(mutation ? "POST" : "GET")( + `${origin}/trpc/${operation}${mutation ? "" : `?input=${encodeURIComponent(encode(body))}`}`, + { + headers: { + authorization: `Bearer ${Redacted.value(input.apiKey)}`, + "content-type": "application/json", + }, + }, + ); + const response = yield* client.execute( + mutation ? HttpClientRequest.bodyText(req, encode(body), "application/json") : req, + ); + if (response.status < 200 || response.status >= 300) + return yield* response.status >= 500 + ? uncertain() + : new KiloCloudError({ operation, reason: "rejected", messageId }); + const bytes = yield* response.stream.pipe( + Stream.runFoldEffect( + () => ({ size: 0, chunks: [] as Uint8Array[] }), + (acc, chunk) => + acc.size + chunk.byteLength > 1024 * 1024 + ? Effect.fail(uncertain()) + : Effect.succeed({ + size: acc.size + chunk.byteLength, + chunks: [...acc.chunks, chunk], + }), + ), + ); + return yield* decodeEnvelope(Buffer.concat(bytes.chunks).toString("utf8")).pipe( + Effect.map((envelope) => envelope.result.data), + ); + }).pipe( + Effect.scoped, + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.provide(FetchHttpClient.layer), + Effect.timeout("30 seconds"), + Effect.mapError((error) => (isCloudError(error) ? error : uncertain())), + ); + }; + + const admit = ( + operation: "start" | "send", + body: unknown, + messageId: string, + sessionId?: string, + ) => + request(operation, body, messageId).pipe( + Effect.flatMap((raw) => + decodeAdmission(raw).pipe( + Effect.mapError( + () => new KiloCloudError({ operation, reason: "admission_unknown", messageId }), + ), + ), + ), + Effect.flatMap((accepted) => + accepted.messageId !== messageId || + (sessionId !== undefined && accepted.cloudAgentSessionId !== sessionId) + ? Effect.fail(new KiloCloudError({ operation, reason: "admission_unknown", messageId })) + : Effect.succeed({ + accountKey: input.accountKey, + sessionId: accepted.cloudAgentSessionId, + messageId, + }), + ), + ); + return { + /** The caller must durably persist this message ID BEFORE submitting. An unknown start has + * no session ID to query through this customer API; operator reconciliation is required. + */ + start: (request: { + readonly messageId: string; + readonly prompt: string; + readonly repository: { + readonly type: "github"; + readonly repo: string; + readonly branch?: string; + }; + readonly model: string; + readonly mode: string; + }) => + admit( + "start", + { + message: { id: request.messageId, prompt: request.prompt }, + repository: request.repository, + agent: { model: request.model, mode: request.mode }, + options: { createdOnPlatform: "kilo-cli" }, + }, + request.messageId, + ), + send: (ref: KiloCloudRef, messageId: string, prompt: string) => + own(ref).pipe( + Effect.andThen( + admit( + "send", + { cloudAgentSessionId: ref.sessionId, message: { id: messageId, prompt } }, + messageId, + ref.sessionId, + ), + ), + ), + result: (ref: KiloCloudRef) => + own(ref).pipe( + Effect.andThen( + request( + "getMessageResult", + { cloudAgentSessionId: ref.sessionId, messageId: ref.messageId }, + ref.messageId, + ), + ), + Effect.flatMap((raw) => + decodeResult(raw).pipe( + Effect.mapError( + () => + new KiloCloudError({ operation: "getMessageResult", reason: "invalid_response" }), + ), + ), + ), + Effect.flatMap((result) => + result.cloudAgentSessionId !== ref.sessionId || result.messageId !== ref.messageId + ? Effect.fail( + new KiloCloudError({ operation: "getMessageResult", reason: "wrong_owner" }), + ) + : Effect.succeed({ ...result, billingStatus: "unknown" as const }), + ), + ), + interrupt: (ref: KiloCloudRef) => + own(ref).pipe( + Effect.andThen( + Effect.fail(new KiloCloudError({ operation: "interrupt", reason: "unsupported" })), + ), + ), + }; +}; diff --git a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts new file mode 100644 index 000000000000..b059ba85c328 --- /dev/null +++ b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts @@ -0,0 +1,114 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Scope from "effect/Scope"; +import { describe } from "vite-plus/test"; + +import * as KiloRuntime from "./KiloRuntime.ts"; + +const binary = process.env.KILO_BIN; +const environment = { + PATH: process.env.PATH, + HTTP_PROXY: process.env.HTTP_PROXY, + HTTPS_PROXY: process.env.HTTPS_PROXY, + NO_PROXY: process.env.NO_PROXY, + NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS, + KILO_DISABLE_MODELS_FETCH: "1", + KILO_DISABLE_DEFAULT_PLUGINS: "1", + KILO_DISABLE_EXTERNAL_SKILLS: "1", + KILO_DISABLE_PROJECT_CONFIG: "1", +}; + +describe.runIf(binary !== undefined)("KiloRuntime native lifecycle", () => { + it.live( + "isolates profiles, closes owned processes and resumes after restart", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-runtime-" }); + const cwd = path.join(root, "work"); + yield* fs.makeDirectory(cwd); + const accountScope = yield* Scope.fork(yield* Effect.scope); + const runtime = yield* KiloRuntime.make({ + instanceId: "personal", + binaryPath: binary!, + profileDirectory: path.join(root, "personal"), + environment, + }).pipe(Effect.provideService(Scope.Scope, accountScope)); + const work = yield* KiloRuntime.make({ + instanceId: "work", + binaryPath: binary!, + profileDirectory: path.join(root, "work-account"), + environment, + }); + const sessionScope = yield* Scope.fork(yield* Effect.scope); + const first = yield* runtime + .open(cwd) + .pipe(Effect.provideService(Scope.Scope, sessionScope)); + const other = yield* work.open(cwd); + const ref = yield* first.client.create([]); + const foreign = { ...ref, instanceId: "work" }; + yield* other.client.read(foreign).pipe(Effect.flip); + const otherRef = yield* other.client.create([]); + yield* Scope.close(sessionScope, Exit.void); + assert.isFalse(yield* first.isRunning); + assert.isTrue(yield* other.isRunning); + const resumed = yield* runtime.open(cwd); + assert.equal((yield* resumed.client.read(ref)).id, ref.sessionId); + yield* Scope.close(accountScope, Exit.void); + assert.isFalse(yield* resumed.isRunning); + const retired = yield* runtime.open(cwd).pipe(Effect.flip); + assert.equal(retired.operation, "open"); + assert.isTrue(yield* other.isRunning); + assert.equal((yield* other.client.read(otherRef)).id, otherRef.sessionId); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + { timeout: 30000 }, + ); + + it.live( + "cleans failed startup and can open a fresh process afterward", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-startup-" }); + const bad = yield* KiloRuntime.make({ + instanceId: "broken", + binaryPath: path.join(root, "missing"), + profileDirectory: root, + environment, + }); + const failure = yield* bad.open(root).pipe(Effect.flip); + assert.equal(failure.operation, "spawn"); + const earlyExit = path.join(root, "early-exit"); + yield* fs.writeFileString( + earlyExit, + "#!/bin/sh\necho do-not-leak-this-diagnostic >&2\nexit 7\n", + ); + yield* fs.chmod(earlyExit, 0o700); + const exiting = yield* KiloRuntime.make({ + instanceId: "early", + binaryPath: earlyExit, + profileDirectory: root, + environment, + }); + const earlyFailure = yield* exiting.open(root).pipe(Effect.flip); + assert.equal(earlyFailure.operation, "startup"); + assert.notInclude(earlyFailure.message, "do-not-leak"); + const good = yield* KiloRuntime.make({ + instanceId: "working", + binaryPath: binary!, + profileDirectory: root, + environment, + }); + const connection = yield* good.open(root); + assert.isTrue(yield* connection.isRunning); + yield* connection.client.create([]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + { timeout: 30000 }, + ); +}); diff --git a/apps/server/src/provider/kilo/KiloRuntime.ts b/apps/server/src/provider/kilo/KiloRuntime.ts new file mode 100644 index 000000000000..4ba712c18f60 --- /dev/null +++ b/apps/server/src/provider/kilo/KiloRuntime.ts @@ -0,0 +1,217 @@ +import * as Context from "effect/Context"; +import * as Crypto from "effect/Crypto"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Encoding from "effect/Encoding"; +import * as Exit from "effect/Exit"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import { resolveSpawnCommand } from "@t3tools/shared/shell"; + +import { signalProcessGroup } from "../../process/processGroup.ts"; +import * as KiloSessionClient from "./KiloSessionClient.ts"; + +export class KiloRuntimeError extends Schema.TaggedError()("KiloRuntimeError", { + operation: Schema.String, + detail: Schema.String, + cause: Schema.optional(Schema.Defect()), +}) { + override get message() { + return this.detail; + } +} + +export interface KiloConnection { + readonly client: Effect.Success>; + readonly stop: Effect.Effect; + readonly cleanup: Effect.Effect; + readonly exitCode: Effect.Effect; + readonly isRunning: Effect.Effect; +} + +export class KiloRuntime extends Context.Service< + KiloRuntime, + { + readonly open: ( + directory: string, + ) => Effect.Effect; + } +>()("t3/provider/kilo/KiloRuntime") {} + +/** Every open owns a process. Registry replacement closes the old account's process scopes. */ +export const make = Effect.fn("KiloRuntime.make")(function* (input: { + readonly instanceId: string; + readonly binaryPath: string; + /** An XDG root for this account, not the Kilo data directory itself. */ + readonly profileDirectory: string; + readonly environment: NodeJS.ProcessEnv; +}) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const platform = yield* HostProcessPlatform; + const owner = yield* Effect.scope; + const fail = (operation: string, detail: string) => (cause: unknown) => + new KiloRuntimeError({ operation, detail, cause }); + const profile = path.resolve(input.profileDirectory); + const environment: NodeJS.ProcessEnv = { + ...input.environment, + XDG_CONFIG_HOME: path.join(profile, "config"), + XDG_DATA_HOME: path.join(profile, "data"), + XDG_CACHE_HOME: path.join(profile, "cache"), + XDG_STATE_HOME: path.join(profile, "state"), + KILO_DISABLE_AUTOUPDATE: "1", + // Background children outlive root turns and need a separate T3 continuation contract. + KILO_EXPERIMENTAL_BACKGROUND_SUBAGENTS: "false", + KILO_SERVER_USERNAME: "kilo", + }; + let closed = false; + yield* Scope.addFinalizer( + owner, + Effect.sync(() => { + closed = true; + }), + ); + // Readiness output can contain project/plugin diagnostics; never include it in client errors. + return KiloRuntime.of({ + open: Effect.fn("KiloRuntime.open")(function* (directory) { + if (closed) + return yield* new KiloRuntimeError({ + operation: "open", + detail: "Kilo account runtime has been retired.", + }); + const caller = yield* Effect.scope; + const scope = yield* Scope.fork(owner); + yield* Scope.addFinalizer(caller, Scope.close(scope, Exit.void)); + const start = Effect.gen(function* () { + for (const name of ["config", "data", "cache", "state"]) { + yield* fs + .makeDirectory(path.join(profile, name), { recursive: true }) + .pipe(Effect.mapError(fail("profile", "Could not prepare the Kilo account profile."))); + } + const password = Encoding.encodeBase64Url( + yield* crypto + .randomBytes(32) + .pipe(Effect.mapError(fail("password", "Could not secure the local Kilo server."))), + ); + const command = yield* resolveSpawnCommand( + input.binaryPath, + ["serve", "--hostname=127.0.0.1", "--port=0"], + { env: environment, extendEnv: false }, + ); + const child = yield* spawner + .spawn( + ChildProcess.make(command.command, command.args, { + cwd: directory, + env: { ...environment, KILO_SERVER_PASSWORD: password }, + extendEnv: false, + detached: platform !== "win32", + shell: command.shell, + }), + ) + .pipe(Effect.mapError(fail("spawn", "Could not start Kilo. Check the binary path."))); + // Only this captured process group is signalled. No process-name matching. + const cleanup = yield* Effect.cached( + Effect.uninterruptible( + platform === "win32" + ? child + .kill({ killSignal: "SIGTERM", forceKillAfter: "1 second" }) + .pipe(Effect.ignore) + : Effect.sync(() => { + try { + signalProcessGroup(Number(child.pid), "SIGTERM"); + } catch { + /* already exited */ + } + }).pipe( + Effect.andThen( + child.exitCode.pipe(Effect.timeoutOption("1 second"), Effect.ignore), + ), + Effect.andThen( + Effect.sync(() => { + try { + signalProcessGroup(Number(child.pid), "SIGKILL"); + } catch { + /* already exited */ + } + }), + ), + ), + ), + ); + yield* Effect.addFinalizer(() => cleanup); + const ready = yield* Deferred.make(); + let output = ""; + yield* child.stdout.pipe( + Stream.decodeText(), + Stream.runForEach((chunk) => { + output = (output + chunk).slice(-65536); + const match = /kilo server listening on (http:\/\/127\.0\.0\.1:\d+)/.exec(output); + return match ? Deferred.succeed(ready, match[1]!).pipe(Effect.asVoid) : Effect.void; + }), + Effect.ignore, + Effect.forkIn(scope), + ); + yield* child.stderr.pipe(Stream.runDrain, Effect.ignore, Effect.forkIn(scope)); + const exitCode = child.exitCode.pipe( + Effect.map(Number), + Effect.orElseSucceed(() => -1), + ); + yield* exitCode.pipe( + Effect.flatMap((code) => + Deferred.fail( + ready, + new KiloRuntimeError({ + operation: "startup", + detail: `Kilo exited during startup (code ${code}).`, + }), + ), + ), + Effect.forkIn(scope), + ); + const url = yield* Deferred.await(ready).pipe( + Effect.timeout("30 seconds"), + Effect.mapError( + fail("startup", "Kilo did not become ready. Check its installation and configuration."), + ), + ); + const client = yield* KiloSessionClient.make({ + instanceId: input.instanceId, + directory, + baseUrl: url, + serverPassword: password, + }).pipe( + Effect.mapError( + fail( + "health", + "Kilo's authenticated health check failed. This provider requires version 7.8.3.", + ), + ), + ); + if (!(yield* child.isRunning.pipe(Effect.orElseSucceed(() => false)))) { + return yield* new KiloRuntimeError({ + operation: "startup", + detail: "Kilo exited before readiness completed.", + }); + } + return { + stop: Scope.close(scope, Exit.void), + cleanup, + client, + exitCode, + isRunning: child.isRunning.pipe(Effect.orElseSucceed(() => false)), + } satisfies KiloConnection; + }); + return yield* start.pipe( + Effect.provideService(Scope.Scope, scope), + Effect.onError(() => Scope.close(scope, Exit.void)), + ); + }), + }); +}); diff --git a/apps/server/src/provider/kilo/KiloSessionClient.test.ts b/apps/server/src/provider/kilo/KiloSessionClient.test.ts index 9c4c8853d87b..46d08ff9c8b9 100644 --- a/apps/server/src/provider/kilo/KiloSessionClient.test.ts +++ b/apps/server/src/provider/kilo/KiloSessionClient.test.ts @@ -57,6 +57,72 @@ async function withClient( } describe("Kilo native SDK boundary", () => { + it("recovers pending interactions only from the verified session family", async () => { + const client = await withClient((req, res) => { + if (req.url?.startsWith("/permission")) + return json(res, [ + { id: "per_root", sessionID: ref.sessionId }, + { id: "per_child", sessionID: "ses_child" }, + { id: "per_foreign", sessionID: "ses_foreign" }, + { id: "per_other_dir", sessionID: "ses_other_dir" }, + ]); + if (req.url?.startsWith("/question")) + return json(res, [{ id: "q_child", sessionID: "ses_grandchild" }]); + const id = req.url?.split("/").at(-1)?.split("?")[0]; + json(res, { + id, + directory: id === "ses_other_dir" ? "/another-workspace" : directory, + ...(id === "ses_child" || id === "ses_other_dir" + ? { parentID: ref.sessionId } + : id === "ses_grandchild" + ? { parentID: "ses_child" } + : {}), + }); + }); + expect((await run(client.pending(ref))).map((item) => item.id)).toEqual(["per_root"]); + expect((await run(client.pending(ref, true))).map((item) => item.id)).toEqual([ + "per_root", + "per_child", + "q_child", + ]); + }); + it.each([null, [], { ses_one: null }, { ses_one: { type: "completed" } }])( + "never treats a malformed native status as idle: %j", + async (payload) => { + const client = await withClient((req, res) => + json( + res, + req.url?.startsWith("/session/status") ? payload : { id: ref.sessionId, directory }, + ), + ); + const failure = await run(client.status(ref).pipe(Effect.flip)); + expect(failure.reason).toBe("invalid_response"); + }, + ); + it.each([{}, { info: {}, parts: null }, { info: {}, parts: [{ type: "text" }] }, null])( + "rejects malformed native generation without a defect: %j", + async (payload) => { + const client = await withClient((req, res) => + json(res, req.method === "GET" ? { id: ref.sessionId, directory } : payload), + ); + const failure = await run( + client.generate(ref, { parts: [{ type: "text", text: "test" }] }).pipe(Effect.flip), + ); + expect(failure.reason).toBe("invalid_response"); + }, + ); + it("classifies a rejected prompt as definitive without resubmitting", async () => { + let submitted = 0; + const client = await withClient((req, res) => { + if (req.method === "GET") return json(res, { id: ref.sessionId, directory }); + submitted++; + res.writeHead(400, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "invalid prompt" })); + }); + const failure = await run(client.prompt(ref, { parts: [] }).pipe(Effect.flip)); + expect(failure.reason).toBe("request_failed"); + expect(submitted).toBe(1); + }); it("uses Kilo auth and lossless directory routing with the real SDK", async () => { const requests: Array<{ url: string; diff --git a/apps/server/src/provider/kilo/KiloSessionClient.ts b/apps/server/src/provider/kilo/KiloSessionClient.ts index 473380409a6b..496c802625e2 100644 --- a/apps/server/src/provider/kilo/KiloSessionClient.ts +++ b/apps/server/src/provider/kilo/KiloSessionClient.ts @@ -1,4 +1,5 @@ -import { createKiloClient, type Event, type KiloClient } from "@kilocode/sdk/v2"; +// @effect-diagnostics globalTimers:off - watchdog for the SDK async iterator; avoids a fiber and timer race per SSE record. +import { createKiloClient, type Event, type KiloClient, type Session } from "@kilocode/sdk/v2"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; @@ -39,6 +40,17 @@ const EventEnvelope = Schema.Struct({ properties: Schema.Record(Schema.String, Schema.Unknown), }); +const isStatusMap = Schema.is( + Schema.Record(Schema.String, Schema.Struct({ type: Schema.Literals(["idle", "busy", "retry"]) })), +); +const isGeneration = Schema.is( + Schema.Struct({ + info: Schema.Record(Schema.String, Schema.Unknown), + parts: Schema.Array( + Schema.Struct({ type: Schema.String, text: Schema.optional(Schema.String) }), + ), + }), +); const isKiloSessionError = Schema.is(KiloSessionError); const isSyncEnvelope = Schema.is( Schema.Struct({ @@ -51,14 +63,12 @@ const isPendingOwners = Schema.is(PendingOwners); const isEventEnvelope = Schema.is(EventEnvelope); const isRecord = Schema.is(Schema.Record(Schema.String, Schema.Unknown)); -function sessionIdOf(event: Event): Effect.Effect { +function sessionIdOf(event: Event): string | undefined | KiloSessionError { // The server also sends replication envelopes omitted from its generated Event union. // Ordinary session/message events follow these and remain the source for this stream. - if (isSyncEnvelope(event)) return Effect.succeed(undefined); + if (isSyncEnvelope(event)) return undefined; if (!isEventEnvelope(event)) { - return Effect.fail( - new KiloSessionError({ operation: "event.subscribe", reason: "invalid_response" }), - ); + return new KiloSessionError({ operation: "event.subscribe", reason: "invalid_response" }); } const properties = event.properties; let value: unknown; @@ -74,20 +84,16 @@ function sessionIdOf(event: Event): Effect.Effect { + const owners = new Map([[ref.sessionId, true]]); + return async (sessionId: string): Promise => { + let id: string | undefined = sessionId; + const visited = new Set(); + while (id && !visited.has(id) && visited.size < 32) { + const cached = owners.get(id); + if (cached !== undefined) { + owners.set(sessionId, cached); + return cached; + } + visited.add(id); + const session: Session | undefined = ( + await client.session.get( + { sessionID: id }, + { + signal: AbortSignal.any([signal, AbortSignal.timeout(10_000)]), + }, + ) + ).data; + if (!isSessionOwner(session) || session.id !== id || session.directory !== input.directory) + break; + id = session.parentID; + } + owners.set(sessionId, false); + return false; + }; + }; + const health = yield* request("global.health", (signal) => client.global.health({ signal })).pipe( Effect.flatMap((value) => decodeHealth(value).pipe( @@ -251,7 +286,7 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { sessionID: ref.sessionId, directory: input.directory, }, - { signal }, + { signal, throwOnError: false }, ), catch: (cause) => new KiloSessionError({ @@ -278,11 +313,47 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { : Effect.fail( new KiloSessionError({ operation: "session.promptAsync", - reason: "invalid_response", + reason: + response.response?.status && + response.response.status >= 400 && + response.response.status < 500 + ? "request_failed" + : "admission_unknown", }), ), ), ), + generate: ( + ref: KiloSessionRef, + prompt: Omit< + Parameters[0], + "sessionID" | "directory" | "workspace" + >, + ) => + read(ref).pipe( + Effect.andThen( + Effect.tryPromise({ + try: (signal) => + client.session.prompt({ ...prompt, sessionID: ref.sessionId }, { signal }), + catch: () => + new KiloSessionError({ operation: "session.prompt", reason: "admission_unknown" }), + }), + ), + Effect.timeout("2 minutes"), + Effect.mapError( + () => new KiloSessionError({ operation: "session.prompt", reason: "request_failed" }), + ), + Effect.flatMap((response) => + isGeneration(response.data) && + response.data && + !response.data.info.error && + response.data.parts.every((part) => part.type !== "text" || typeof part.text === "string") + ? Effect.succeed(response.data) + : Effect.fail( + new KiloSessionError({ operation: "session.prompt", reason: "invalid_response" }), + ), + ), + ), abort: (ref: KiloSessionRef) => owned(ref, "session.abort", (signal) => client.session.abort({ sessionID: ref.sessionId }, { signal }), @@ -325,7 +396,50 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { ), ), ), - events: (ref: KiloSessionRef) => + pending: (ref: KiloSessionRef, includeChildren = false) => + owned(ref, "interaction.list", async (signal) => { + const [permissions, questions] = await Promise.all([ + client.permission.list(undefined, { signal }), + client.question.list(undefined, { signal }), + ]); + if (!isPendingOwners(permissions.data) || !isPendingOwners(questions.data)) + throw new KiloSessionError({ operation: "interaction.list", reason: "invalid_response" }); + const belongs = ownerCheck(ref, signal); + const pending = [...permissions.data, ...questions.data]; + const decisions = await Promise.all( + pending.map((entry) => + entry.sessionID === ref.sessionId + ? true + : includeChildren + ? belongs(entry.sessionID) + : false, + ), + ); + return { data: pending.filter((_, index) => decisions[index]) }; + }), + status: (ref: KiloSessionRef) => + owned(ref, "session.status", (signal) => client.session.status(undefined, { signal })).pipe( + Effect.filterOrFail( + isStatusMap, + () => new KiloSessionError({ operation: "session.status", reason: "invalid_response" }), + ), + Effect.map((statuses) => statuses[ref.sessionId]?.type ?? "idle"), + ), + setPermissions: ( + ref: KiloSessionRef, + permission: NonNullable[0]>["permission"], + ) => + owned(ref, "session.update", (signal) => + client.session.update( + { sessionID: ref.sessionId, ...(permission === undefined ? {} : { permission }) }, + { signal }, + ), + ), + events: ( + ref: KiloSessionRef, + onConnected: Effect.Effect = Effect.void, + includeChildren = false, + ) => Stream.unwrap( read(ref).pipe( Effect.andThen( @@ -352,8 +466,36 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { const interruptible: AsyncIterable = { [Symbol.asyncIterator]() { const iterator = subscription.stream[Symbol.asyncIterator](); + const belongsToRoot = ownerCheck(ref, controller.signal); return { - next: () => iterator.next(), + next: async () => { + // Filter before crossing the Effect stream boundary. No batching or additional + // queue: SDK backpressure and order are preserved, including readiness. + while (true) { + const timer = setTimeout(() => controller.abort(), 45_000); + let result: IteratorResult; + try { + result = await iterator.next(); + } finally { + clearTimeout(timer); + } + if (result.done) return result; + if ( + isEventEnvelope(result.value) && + result.value.type === "server.connected" + ) + return result; + const owner = sessionIdOf(result.value); + if (typeof owner === "object") throw owner; + if ( + owner === ref.sessionId || + (includeChildren && + typeof owner === "string" && + (await belongsToRoot(owner))) + ) + return result; + } + }, return: async () => { // Abort a pending reader.read before awaiting generator cleanup. // A scope finalizer alone runs after fromAsyncIterable's finalizer. @@ -365,16 +507,15 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { }; }, }; - return Stream.fromAsyncIterable( - interruptible, - (cause) => - new KiloSessionError({ - operation: "event.subscribe", - reason: "request_failed", - cause, - }), + return Stream.fromAsyncIterable(interruptible, (cause) => + isKiloSessionError(cause) + ? cause + : new KiloSessionError({ + operation: "event.subscribe", + reason: "request_failed", + cause, + }), ).pipe( - Stream.timeout("45 seconds"), Stream.mapError((cause) => isKiloSessionError(cause) ? cause @@ -384,9 +525,12 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { cause, }), ), - Stream.filterEffect((event) => - sessionIdOf(event).pipe(Effect.map((sessionId) => sessionId === ref.sessionId)), + Stream.tap((event) => + isEventEnvelope(event) && event.type === "server.connected" + ? onConnected + : Effect.void, ), + Stream.filter((event) => event.type !== "server.connected"), Stream.concat( Stream.unwrap( Effect.sync(() => diff --git a/apps/server/src/textGeneration/KiloTextGeneration.ts b/apps/server/src/textGeneration/KiloTextGeneration.ts new file mode 100644 index 000000000000..75481b55b804 --- /dev/null +++ b/apps/server/src/textGeneration/KiloTextGeneration.ts @@ -0,0 +1,68 @@ +import { TextGenerationError } from "@t3tools/contracts"; +import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; +import { extractJsonObject } from "@t3tools/shared/schemaJson"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import { resolveAttachmentPath } from "../attachmentStore.ts"; +import { toOpenCodeFileParts } from "../provider/opencodeRuntime.ts"; +import type * as KiloRuntime from "../provider/kilo/KiloRuntime.ts"; +import { makeOpenCodeOperations, type OpenCodeJsonRunner } from "./OpenCodeTextGeneration.ts"; + +/** Only prompt construction is shared. Protocol, credentials and lifetime belong to Kilo. */ +export function make(runtime: KiloRuntime.KiloRuntime["Service"], attachmentsDir?: string) { + const run: OpenCodeJsonRunner = (input) => + Effect.gen(function* () { + const separator = input.modelSelection.model.indexOf("/"); + if (separator <= 0) + return yield* new TextGenerationError({ + operation: input.operation, + detail: "Kilo models must use provider/model format.", + }); + if (input.attachments?.length && !attachmentsDir) + return yield* new TextGenerationError({ + operation: input.operation, + detail: "Kilo text generation attachments are not configured.", + }); + const connection = yield* runtime.open(input.cwd); + const ref = yield* connection.client.create([ + { permission: "*", pattern: "*", action: "deny" }, + ]); + const agent = getModelSelectionStringOptionValue(input.modelSelection, "agent"); + const variant = getModelSelectionStringOptionValue(input.modelSelection, "variant"); + const response = yield* connection.client.generate(ref, { + ...(agent ? { agent } : {}), + ...(variant ? { variant } : {}), + model: { + providerID: input.modelSelection.model.slice(0, separator), + modelID: input.modelSelection.model.slice(separator + 1), + }, + parts: [ + { type: "text", text: input.prompt }, + ...toOpenCodeFileParts({ + attachments: input.attachments, + resolveAttachmentPath: (attachment) => + attachmentsDir ? resolveAttachmentPath({ attachmentsDir, attachment }) : null, + }), + ], + }); + const text = response.parts + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("\n"); + // The operation supplies its output schema; it cannot be compiled once at module scope. + // oxlint-disable-next-line t3code/no-inline-schema-compile + return yield* Schema.decodeUnknownEffect(Schema.fromJsonString(input.outputSchemaJson))( + extractJsonObject(text), + ); + }).pipe( + Effect.scoped, + Effect.mapError( + () => + new TextGenerationError({ + operation: input.operation, + detail: "Kilo text generation failed. The request was not retried.", + }), + ), + ); + return makeOpenCodeOperations(run); +} diff --git a/apps/web/src/components/Icons.tsx b/apps/web/src/components/Icons.tsx index a940d2ae90fb..b08692066356 100644 --- a/apps/web/src/components/Icons.tsx +++ b/apps/web/src/components/Icons.tsx @@ -1,3 +1,4 @@ +import { kiloIconPaths } from "@t3tools/client-runtime/kilo-icon"; import React, { type SVGProps, useId } from "react"; import { cn } from "~/lib/utils"; export type Icon = React.FC>; @@ -734,3 +735,11 @@ export const ComputerUseAppIcon: Icon = (props) => { ); }; + +export const KiloIcon: Icon = (props) => ( + + {kiloIconPaths.map((path) => ( + + ))} + +); diff --git a/apps/web/src/components/chat/ProviderInstanceIcon.tsx b/apps/web/src/components/chat/ProviderInstanceIcon.tsx index 1bb318376d23..e2beabd6e58b 100644 --- a/apps/web/src/components/chat/ProviderInstanceIcon.tsx +++ b/apps/web/src/components/chat/ProviderInstanceIcon.tsx @@ -11,6 +11,7 @@ import { Icon, OpenAI, OpenCodeIcon, + KiloIcon, PiAgentIcon, } from "../Icons"; @@ -25,6 +26,7 @@ const PROVIDER_ICON_BY_PROVIDER: Partial> = { [ProviderDriverKind.make("codex")]: OpenAI, [ProviderDriverKind.make("claudeAgent")]: ClaudeAI, [ProviderDriverKind.make("opencode")]: OpenCodeIcon, + [ProviderDriverKind.make("kilo")]: KiloIcon, [ProviderDriverKind.make("cursor")]: CursorIcon, [ProviderDriverKind.make("grok")]: GrokIcon, [ProviderDriverKind.make("antigravity")]: AntigravityIcon, diff --git a/apps/web/src/components/settings/providerDriverMeta.ts b/apps/web/src/components/settings/providerDriverMeta.ts index 08f9d83333c1..7fb52e849e7c 100644 --- a/apps/web/src/components/settings/providerDriverMeta.ts +++ b/apps/web/src/components/settings/providerDriverMeta.ts @@ -6,6 +6,7 @@ import { CursorSettings, GrokSettings, OpenCodeSettings, + KiloSettings, PiSettings, ProviderDriverKind, } from "@t3tools/contracts"; @@ -47,6 +48,22 @@ export interface ProviderEnvironmentFieldDefinition { } const PROVIDER_CLIENT_DEFINITIONS: readonly ProviderClientDefinition[] = [ + { + value: ProviderDriverKind.make("kilo"), + label: "Kilo", + settingsSchema: KiloSettings, + hasDefaultInstance: false, + badgeLabel: "Preview", + environmentFields: [ + { + name: "KILO_API_KEY", + label: "Kilo API key", + sensitive: true, + description: + "Optional existing key for this account profile. Stored with this provider's environment.", + }, + ], + }, { value: ProviderDriverKind.make("codex"), label: "Codex", diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md new file mode 100644 index 000000000000..18c97b2846d3 --- /dev/null +++ b/docs/user/providers-kilo.md @@ -0,0 +1,37 @@ +# Kilo + +The native Kilo provider is a preview. Install Kilo CLI **7.8.3** on the machine +running the T3 environment, then add Kilo in **Settings > Providers**. Set the +binary path if `kilo` is not on that machine's PATH. Other CLI versions are +rejected because their protocol has not been verified with this provider. + +Each provider instance has a separate account profile. T3 does not copy credentials +from your ordinary Kilo installation. You can supply an existing `KILO_API_KEY` +in the instance's environment settings or select an existing profile directory. +That directory must contain the `config`, `data`, `cache`, and `state` directories +used as Kilo's XDG roots. It is not the directory containing `auth.json` alone. + +Use a separate instance and profile for each account. Changing the account profile +or instance environment retires its running processes. Existing threads cannot +resume under the replacement account. A Ready status confirms local CLI readiness; +it does not prove that a model account is authenticated or has available credit. + +Prompts run in the selected T3 workspace. Use separate T3 worktrees for tasks that +must not share files. Separate conversation IDs alone do not isolate a checkout. +Stop terminates the task's owned local process group. A later prompt can restore +its saved conversation in a new process. Rewind copies the retained native +conversation and lets T3 restore the selected file checkpoint. + +Subagents require Full access with no additional approval or sandbox restrictions. +Kilo's own configured child approvals still apply. Restricted modes and Plan mode +block subagent creation because Kilo 7.8.3 does not propagate T3's approval rules +to children. Background subagents and independent child cancellation are not +supported. Tool results appear when the tool finishes; live tool output is not +advertised. + +You can select models and control tasks from web, desktop, and mobile clients +connected to the environment. Configure account profiles in web or desktop +settings. A disconnected client does not stop its task. + +Kilo Cloud Agents are not available in this preview. No cloud task is launched and +no repository is uploaded by selecting this provider. diff --git a/knip.jsonc b/knip.jsonc index 8865bf58869c..cb2ab91e03d1 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -25,6 +25,7 @@ "scripts/cli.ts", "scripts/evaluate-thread-titles.ts", "scripts/measure-pr-preview.ts", + "scripts/kilo-stream-benchmark.mjs", "scripts/probe-claude-fork-local-rollback-replay.ts", "scripts/update-test-shard-weights.ts", "scripts/verify-background-live.ts", @@ -38,6 +39,7 @@ // Electron loads these bundles by filename rather than importing them. "entry": [ "gnome-extension/extension.js!", + "scripts/kilo-ui-evidence.mjs", "src/electron/WindowsForegroundFocusWorker.ts!", "src/snapShot/GlobalShiftShortcutWorker.ts!", "src/snapShot/RegionSnapShotWorker.ts!", diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 087d29f698d7..40add7a6bc51 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -362,6 +362,10 @@ "./device/duo-control": { "types": "./src/device/duoControl.ts", "default": "./src/device/duoControl.ts" + }, + "./kilo-icon": { + "types": "./src/kiloIcon.ts", + "default": "./src/kiloIcon.ts" } }, "scripts": { diff --git a/packages/client-runtime/src/kiloIcon.ts b/packages/client-runtime/src/kiloIcon.ts new file mode 100644 index 000000000000..71dca91f034d --- /dev/null +++ b/packages/client-runtime/src/kiloIcon.ts @@ -0,0 +1,8 @@ +/** Official Kilo Code mark, v7.8.3, MIT. See third-party-licenses.config.json. */ +export const kiloIconPaths = [ + { d: "M512 0H0V512H512V0Z", fill: "black" }, + { + d: "M322 377H377V421H307.857L278 391.143V322H322V377ZM421 307.857L391.143 278H322V322L377 322V377H421V307.857ZM234 278H190V322H234V278ZM91 391.143L120.857 421H234V377H135V278H91V391.143ZM371.172 189.999V120.856L341.315 90.9995H278V135H327.172V189.999H278V233.999H421V189.999H371.172ZM135 91H91V233.999H135V184.5H190V233.999H234V184.5L190 140.5H135V91ZM234 91H190V140.5H234V91Z", + fill: "#FAF74F", + }, +] as const; diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 1820008b77dd..fa6c6c44ba8d 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -796,6 +796,30 @@ export type OrchestrationV2PendingBackgroundTask = typeof OrchestrationV2Pending /** Provider and adapter metadata that should not overwrite the app thread's title. */ export const OrchestrationV2ProviderThreadNativeMetadata = Schema.Struct({ + /** Native sessions may only be resumed within the account/configuration that created them. */ + continuationKey: Schema.optional(TrimmedNonEmptyString), + /** Providers that copy history with fresh message IDs preserve durable turn boundaries here. */ + messageAliases: Schema.optional(Schema.Record(Schema.String, Schema.String)), + /** Native user messages already have an app ID when submitted by T3. */ + turnCorrelations: Schema.optional( + Schema.Record( + Schema.String, + Schema.Struct({ + messageId: MessageId, + attachments: Schema.optional(Schema.Array(ChatAttachment)), + createdBy: Schema.optional(OrchestrationV2Actor), + creationSource: Schema.optional(OrchestrationV2CreationSource), + scheduledTaskId: Schema.optional(ScheduledTaskId), + senderThreadId: Schema.optional(ThreadId), + nodeId: NodeId, + runId: RunId, + attemptId: RunAttemptId, + ordinal: PositiveInt, + terminalStatus: Schema.optional(Schema.Literals(["completed", "failed", "interrupted"])), + completedAt: Schema.optional(TrimmedNonEmptyString), + }), + ), + ), /** Provider-reported selection for display, separate from the app's saved preferences. */ modelSelection: Schema.optional(ModelSelection), title: Schema.optional(Schema.NullOr(TrimmedNonEmptyString)), diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index e84ed36c9718..4e5f00742477 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -968,6 +968,41 @@ export const OpenCodeSettings = makeProviderSettingsSchema( ); export type OpenCodeSettings = typeof OpenCodeSettings.Type; +export const KiloSettings = makeProviderSettingsSchema( + { + enabled: Schema.Boolean.pipe( + Schema.withDecodingDefault(Effect.succeed(false)), + Schema.annotateKey({ providerSettingsForm: { hidden: true } }), + ), + binaryPath: makeBinaryPathSetting("kilo").pipe( + Schema.annotateKey({ + title: "Binary path", + description: "Kilo CLI 7.8.3 executable.", + providerSettingsForm: { placeholder: "kilo", clearWhenEmpty: "omit" }, + }), + ), + accountId: TrimmedNonEmptyString.pipe( + Schema.withDecodingDefault(Effect.succeed("default")), + Schema.annotateKey({ + title: "Account profile", + description: + "Use a separate profile for each account. Changing it disconnects the previous account's sessions.", + }), + ), + profileDirectory: TrimmedString.pipe( + Schema.withDecodingDefault(Effect.succeed("")), + Schema.annotateKey({ + title: "Profile directory", + description: + "Optional XDG root containing config, data, cache and state. Leave empty for an isolated T3 profile. Sign in with Kilo separately; T3 never signs in automatically.", + providerSettingsForm: { placeholder: "Managed by T3", clearWhenEmpty: "omit" }, + }), + ), + }, + { order: ["binaryPath", "accountId", "profileDirectory"] }, +); +export type KiloSettings = typeof KiloSettings.Type; + /** * A read-only quota source outside this environment's provider CLIs. The * only kind today is a CLIProxyAPI hub, whose management API reports the diff --git a/third-party-licenses.config.json b/third-party-licenses.config.json index 7e9e4f5db83c..4afeb591c312 100644 --- a/third-party-licenses.config.json +++ b/third-party-licenses.config.json @@ -149,6 +149,19 @@ "name": "Expo text editor implementation", "noticeFile": "apps/mobile/modules/t3-composer-editor/LICENSE", "sourceUrl": "https://github.com/expo/expo" + }, + { + "bundles": ["assets", "desktop", "web", "mobile"], + "license": "MIT", + "name": "Kilo Code icon", + "sourceUrl": "https://github.com/Kilo-Org/kilo/blob/v7.8.3/packages/kilo-vscode/assets/icons/kilo-light.svg", + "version": "7.8.3", + "generatedNotices": [ + { + "licenseId": "MIT", + "copyrights": ["Copyright (c) 2026 Kilo Code", "Copyright (c) 2025 opencode"] + } + ] } ], "packageOverrides": [ @@ -599,6 +612,17 @@ "licenseId": "MIT", "copyrights": ["Copyright (c) 2015-present 650 Industries, Inc. (aka Expo)"] } + }, + { + "name": "@kilocode/sdk", + "version": "7.8.3", + "generatedNotice": { + "licenseId": "MIT", + "copyrights": ["Copyright (c) 2026 Kilo Code", "Copyright (c) 2025 opencode"], + "preamble": [ + "License from Kilo-Org/kilo v7.8.3, commit 59f1428abb5fe782ee7bd4d258e72a08b74aadb4. The published SDK tarball omits the root LICENSE file." + ] + } } ] } From 98ad7f50ed3af0af7537152d8ff067a2c1f6acd4 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 11:49:44 +0000 Subject: [PATCH 04/44] ci: identify the Kilo fork by repository ID --- .github/workflows/kilo-provider.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/kilo-provider.yml b/.github/workflows/kilo-provider.yml index 13b3a305df4b..c159b77fd377 100644 --- a/.github/workflows/kilo-provider.yml +++ b/.github/workflows/kilo-provider.yml @@ -31,7 +31,8 @@ jobs: provider-integration: # The upstream suite uses Blacksmith runners unavailable to this fork. # Reassess this workflow before proposing the provider upstream. - if: github.repository == 'Githubguy132010/t3code' + # The owner's former login redirects; the repository ID survives that rename. + if: github.repository_id == '1286185343' runs-on: ubuntu-24.04 timeout-minutes: 25 steps: @@ -118,6 +119,14 @@ jobs: KILO_EVIDENCE_DIR: ${{ runner.temp }}/kilo-evidence run: node apps/desktop/scripts/kilo-ui-evidence.mjs + - name: Compare equivalent stream workloads against the unchanged SDK layer + run: | + mkdir -p "$RUNNER_TEMP/kilo-baseline" "$RUNNER_TEMP/kilo-evidence" + git fetch --no-tags --depth=1 origin dfa68127e1a4a992b4184babe0e193432465bdd2 + git show dfa68127e1a4a992b4184babe0e193432465bdd2:apps/server/src/provider/kilo/KiloSessionClient.ts > "$RUNNER_TEMP/kilo-baseline/KiloSessionClient.ts" + ln -s "$GITHUB_WORKSPACE/apps/server/node_modules" "$RUNNER_TEMP/kilo-baseline/node_modules" + node --expose-gc apps/server/scripts/kilo-stream-benchmark.mjs "$RUNNER_TEMP/kilo-baseline/KiloSessionClient.ts" > "$RUNNER_TEMP/kilo-evidence/stream-benchmark.json" + - uses: actions/upload-artifact@v4 if: always() with: From 3a705c74a08b78162ba6a68d97d79d59d9f75421 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 11:57:00 +0000 Subject: [PATCH 05/44] ci: install Linux desktop build dependencies --- .github/workflows/kilo-provider.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/kilo-provider.yml b/.github/workflows/kilo-provider.yml index c159b77fd377..820734592a25 100644 --- a/.github/workflows/kilo-provider.yml +++ b/.github/workflows/kilo-provider.yml @@ -101,7 +101,9 @@ jobs: - name: Build web and desktop bundles run: | - vp run --filter @t3tools/web build + sudo apt-get update + sudo apt-get install -y pkg-config libsecret-1-dev + # The desktop build includes its web and server dependencies. vp run --filter @t3tools/desktop build - name: Export the Android JavaScript bundle From c406f9601b5f7e19fbbe1fa92b5f6215852599b2 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 12:01:45 +0000 Subject: [PATCH 06/44] test(provider): verify all Kilo text generation operations --- .../Adapters/KiloAdapterV2.live.test.ts | 50 ++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts index d257b0b23e03..9f3225e5961c 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts @@ -46,8 +46,10 @@ const inference = Effect.acquireRelease( const requests: Array> = []; const control: { mode: "text" | "approval" | "question" | "json" | "subagent" | "subagent-approval"; + json: string; } = { mode: "text", + json: '{"title":"Local fixture title"}', }; const server = NodeHttp.createServer((req, res) => { let body = ""; @@ -194,7 +196,7 @@ const inference = Effect.acquireRelease( `data: ${JSON.stringify({ id: "chatcmpl-local", object: "chat.completion.chunk", created: 0, model: "test", choices: [{ index: 0, delta: { reasoning_content: "Fixture reasoning." }, finish_reason: null }] })}\n\n`, ); for (const text of control.mode === "json" - ? ['{"title":"Local fixture title"}'] + ? [control.json] : ["Hello ", "from ", "local Kilo."]) res.write( `data: ${JSON.stringify({ @@ -633,12 +635,56 @@ describe.runIf(binary !== undefined)("Kilo adapter with native runtime and local assert.isFalse(seen.slice(before).some((event) => event.type === "app_thread.created")); } model.control.mode = "json"; - const generated = yield* KiloTextGeneration.make(runtime).generateThreadTitle({ + const textGeneration = KiloTextGeneration.make(runtime); + const generated = yield* textGeneration.generateThreadTitle({ cwd: root, modelSelection, message: "Name this test thread", }); assert.equal(generated.title, "Local fixture title"); + model.control.json = '{"branch":" Local Kilo "}'; + const branch = yield* textGeneration.generateBranchName({ + cwd: root, + modelSelection, + message: "Add the local Kilo provider", + }); + assert.equal(branch.branch, "local-kilo"); + model.control.json = + '{"subject":"feat: add local Kilo","body":" Connect the native runtime. ","branch":"local-kilo"}'; + const commit = yield* textGeneration.generateCommitMessage({ + cwd: root, + modelSelection, + branch: "main", + stagedSummary: "1 file changed", + stagedPatch: "+local Kilo provider", + includeBranch: true, + }); + assert.deepEqual(commit, { + subject: "feat: add local Kilo", + body: "Connect the native runtime.", + branch: "feature/local-kilo", + }); + model.control.json = + '{"title":"feat: add local Kilo","body":" Add an isolated native runtime. "}'; + const pr = yield* textGeneration.generatePrContent({ + cwd: root, + modelSelection, + baseBranch: "main", + headBranch: "local-kilo", + commitSummary: "feat: add local Kilo", + diffSummary: "1 file changed", + diffPatch: "+local Kilo provider", + }); + assert.deepEqual(pr, { + title: "feat: add local Kilo", + body: "Add an isolated native runtime.", + }); + // Native text may be valid JSON without satisfying the requested output schema. + // Reject it at the Kilo boundary rather than handing malformed results to T3. + const malformed = yield* textGeneration + .generateBranchName({ cwd: root, modelSelection, message: "Name the branch" }) + .pipe(Effect.flip); + assert.equal(malformed.operation, "generateBranchName"); model.control.mode = "text"; const workspace = path.join(root, "integration-workspace"); yield* fs.makeDirectory(workspace); From fee1809f5d51c99238cb5cf80eb6adca9e2901bb Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 16:02:46 +0000 Subject: [PATCH 07/44] feat(provider): connect isolated Kilo Cloud tasks to orchestration --- .github/workflows/kilo-provider.yml | 16 +- apps/desktop/scripts/kilo-ui-evidence.mjs | 65 +- apps/mobile/src/components/ProviderIcon.tsx | 2 +- .../SettingsProviderAccountsRouteScreen.tsx | 9 +- .../features/threads/ThreadDetailScreen.tsx | 27 +- .../features/threads/ThreadRouteScreen.tsx | 60 +- apps/mobile/src/state/entities.ts | 8 + .../Adapters/KiloCloudAdapterV2.live.test.ts | 459 ++++++ .../Adapters/KiloCloudAdapterV2.test.ts | 586 ++++++++ .../Adapters/KiloCloudAdapterV2.ts | 1247 +++++++++++++++++ .../src/orchestration-v2/EffectOutbox.ts | 2 + .../src/orchestration-v2/EffectWorker.ts | 8 +- .../src/orchestration-v2/ProviderAdapter.ts | 2 + .../ProviderRuntimeRecoveryService.test.ts | 131 ++ .../ProviderRuntimeRecoveryService.ts | 111 ++ .../ProviderSessionManager.ts | 30 +- .../ProviderTurnControlService.ts | 8 + .../ProviderTurnStartService.ts | 35 +- .../RunExecutionService.test.ts | 40 + .../orchestration-v2/RunExecutionService.ts | 175 ++- .../src/provider/Drivers/KiloCloudDriver.ts | 180 +++ apps/server/src/provider/builtInDrivers.ts | 5 +- .../src/provider/kilo/KiloCloudAccount.ts | 55 + .../src/provider/kilo/KiloCloudJournal.ts | 126 ++ .../kilo/KiloCloudWebClient.live.test.ts | 44 + .../provider/kilo/KiloCloudWebClient.test.ts | 343 +++++ .../src/provider/kilo/KiloCloudWebClient.ts | 605 ++++++++ apps/web/src/components/ChatView.tsx | 424 +++--- .../components/chat/ProviderInstanceIcon.tsx | 1 + .../components/settings/providerDriverMeta.ts | 8 + docs/user/providers-kilo.md | 29 +- packages/contracts/src/orchestrationV2.ts | 38 + packages/contracts/src/settings.ts | 47 + 33 files changed, 4631 insertions(+), 295 deletions(-) create mode 100644 apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.live.test.ts create mode 100644 apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts create mode 100644 apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts create mode 100644 apps/server/src/provider/Drivers/KiloCloudDriver.ts create mode 100644 apps/server/src/provider/kilo/KiloCloudAccount.ts create mode 100644 apps/server/src/provider/kilo/KiloCloudJournal.ts create mode 100644 apps/server/src/provider/kilo/KiloCloudWebClient.live.test.ts create mode 100644 apps/server/src/provider/kilo/KiloCloudWebClient.test.ts create mode 100644 apps/server/src/provider/kilo/KiloCloudWebClient.ts diff --git a/.github/workflows/kilo-provider.yml b/.github/workflows/kilo-provider.yml index 820734592a25..309b3fce72c0 100644 --- a/.github/workflows/kilo-provider.yml +++ b/.github/workflows/kilo-provider.yml @@ -5,9 +5,9 @@ on: paths: - .github/workflows/kilo-provider.yml - apps/server/src/provider/kilo/** - - apps/server/src/provider/Drivers/KiloDriver.ts + - apps/server/src/provider/Drivers/Kilo*Driver.ts - apps/server/src/provider/builtInDrivers.ts - - apps/server/src/orchestration-v2/Adapters/KiloAdapterV2* + - apps/server/src/orchestration-v2/** - apps/server/src/textGeneration/KiloTextGeneration.ts - apps/server/scripts/kilo-stream-benchmark.mjs - apps/desktop/scripts/kilo-ui-evidence.mjs @@ -54,6 +54,10 @@ jobs: run: >- vp lint apps/server/src/provider/kilo apps/server/src/provider/Drivers/KiloDriver.ts + apps/server/src/provider/Drivers/KiloCloudDriver.ts + apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts + apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts + apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.live.test.ts apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts apps/server/src/textGeneration/KiloTextGeneration.ts @@ -88,9 +92,15 @@ jobs: KILO_BIN: ${{ runner.temp }}/kilo-probe/node_modules/.bin/kilo KILO_TEST_ROOT: ${{ runner.temp }} run: >- - vp test run + vp test run --no-file-parallelism apps/server/src/provider/kilo apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts + apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts + apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts + apps/server/src/orchestration-v2/ProviderSessionManager.test.ts + apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts + apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts + apps/server/src/orchestration-v2/RunExecutionService.test.ts apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.test.ts apps/server/src/provider/opencodeRuntime.environment.test.ts apps/server/src/provider/opencodeRuntime.permissions.test.ts diff --git a/apps/desktop/scripts/kilo-ui-evidence.mjs b/apps/desktop/scripts/kilo-ui-evidence.mjs index 8792d6c4312f..17fd657a2451 100644 --- a/apps/desktop/scripts/kilo-ui-evidence.mjs +++ b/apps/desktop/scripts/kilo-ui-evidence.mjs @@ -115,6 +115,19 @@ await NodeFSP.writeFile( ]), ), providerInstances: { + kiloCloud: { + driver: "kilo-cloud", + displayName: "Kilo Cloud", + enabled: !!process.env.KILO_CLOUD_TEST_PROFILE, + config: { + enabled: !!process.env.KILO_CLOUD_TEST_PROFILE, + profileDirectory: process.env.KILO_CLOUD_TEST_PROFILE ?? "", + repository: "thomasbrugman/t3-kilo-cloud-test", + branch: "main", + model: "deepseek/deepseek-v4.1-flash", + cloudConsent: !!process.env.KILO_CLOUD_TEST_PROFILE, + }, + }, kilo: { driver: "kilo", displayName: "Kilo", @@ -182,17 +195,41 @@ try { await page.getByText("Local folder", { exact: true }).click(); await page.getByPlaceholder("Enter path (e.g. ~/projects/my-app)").fill(workspace); await page.getByPlaceholder("Enter path (e.g. ~/projects/my-app)").press("Enter"); + if (process.env.KILO_CLOUD_TEST_PROFILE) { + await page.locator("[data-chat-provider-model-picker-label]").click(); + await page.getByPlaceholder("Search models...").fill("deepseek-v4.1-flash"); + await page.getByText("deepseek/deepseek-v4.1-flash", { exact: true }).last().click(); + await page + .getByText("Closing T3 does not stop remote work or billing.", { exact: false }) + .waitFor(); + await page.getByRole("button", { name: "Unknown", exact: true }).click(); + await page.getByRole("menuitemradio", { name: /^Low/ }).click(); + await page.getByRole("button", { name: "Low", exact: true }).waitFor(); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "cloud-before-send.png"), + }); + } await page.locator("[data-chat-provider-model-picker-label]").click(); await page.getByPlaceholder("Search models...").fill("Local fixture"); await page.getByText("Local fixture", { exact: true }).last().click(); await page.getByRole("button", { name: "Local fixture", exact: true }).waitFor(); await page.locator("[contenteditable=true]").fill("Kilo local integration: say hello."); - await page.screenshot({ path: NodePath.join(evidence, "before-send.png") }); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "before-send.png"), + }); await page.getByRole("button", { name: "Submit message", exact: true }).click(); await page.getByText(answer, { exact: true }).waitFor({ timeout: 60000 }); - await page.screenshot({ path: NodePath.join(evidence, "streamed-answer.png") }); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "streamed-answer.png"), + }); await page.getByRole("button", { name: "Submit message", exact: true }).waitFor(); - await page.screenshot({ path: NodePath.join(evidence, "completed-answer.png") }); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "completed-answer.png"), + }); console.log("Local native answer rendered; opening provider settings."); await page.getByRole("button", { name: "Settings", exact: true }).click(); await page.waitForURL("**/settings/general*"); @@ -200,7 +237,27 @@ try { await page.getByRole("button", { name: "Providers", exact: true }).click(); await page.waitForURL("**/settings/providers*"); await page.getByRole("button", { name: "Add provider", exact: true }).waitFor(); - await page.screenshot({ path: NodePath.join(evidence, "provider-settings.png") }); + if (process.env.KILO_CLOUD_TEST_PROFILE) { + const consent = page.getByRole("switch", { name: "Allow paid cloud execution", exact: true }); + await consent.click(); + await page.waitForFunction( + () => + document + .querySelector('[role="switch"][aria-label="Allow paid cloud execution"]') + ?.getAttribute("aria-checked") === "false", + ); + await consent.click(); + await page.waitForFunction( + () => + document + .querySelector('[role="switch"][aria-label="Allow paid cloud execution"]') + ?.getAttribute("aria-checked") === "true", + ); + } + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "provider-settings.png"), + }); await context.close(); if (!requests) throw new Error("The real CLI did not contact the local inference fixture"); await NodeFSP.writeFile( diff --git a/apps/mobile/src/components/ProviderIcon.tsx b/apps/mobile/src/components/ProviderIcon.tsx index bed044599bfc..61178d0ea75c 100644 --- a/apps/mobile/src/components/ProviderIcon.tsx +++ b/apps/mobile/src/components/ProviderIcon.tsx @@ -134,7 +134,7 @@ export function ProviderIcon(props: ProviderIconProps) { ); } - if (props.provider === "kilo") { + if (props.provider === "kilo" || props.provider === "kilo-cloud") { return ( {kiloIconPaths.map((path) => ( diff --git a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx index 5fda519517ae..dc4fdd7bd55c 100644 --- a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx @@ -44,20 +44,22 @@ export function SettingsProviderAccountsRouteScreen() { {environment.serverConfig.providers .filter( (provider) => + provider.driver === "kilo-cloud" || provider.driver === "kilo" || provider.setup?.canAuthenticate || (provider.driver === "acpRegistry" && provider.installed), ) .map((provider) => - provider.driver === "kilo" ? ( + provider.driver === "kilo" || provider.driver === "kilo-cloud" ? ( {provider.displayName} - {provider.message ?? - "Kilo runs on this environment. Manage its isolated account profile in web or desktop Settings."} + {(provider.message ?? provider.driver === "kilo-cloud") + ? "Kilo Cloud runs in a remote repository and uses Kilo credit. Closing T3 does not stop remote work or billing. Manage its account and repository in web or desktop Settings." + : "Kilo runs on this environment. Manage its isolated account profile in web or desktop Settings."} ) : ( @@ -70,6 +72,7 @@ export function SettingsProviderAccountsRouteScreen() { )} {!environment.serverConfig.providers.some( (provider) => + provider.driver === "kilo-cloud" || provider.driver === "kilo" || provider.setup?.canAuthenticate || (provider.driver === "acpRegistry" && provider.installed), diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index 623ae734053d..a0593c69c1d2 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -1,4 +1,5 @@ -import { useThreadReportedModelSelection } from "../../state/entities"; +import { AppText } from "../../components/AppText"; +import { useThreadCloudExecution, useThreadReportedModelSelection } from "../../state/entities"; import { UsageLimitRecoveryCard } from "./UsageLimitRecoveryCard"; import { useNavigation } from "@react-navigation/native"; import type { WorktreeSetupCardProps } from "./worktree-setup-card"; @@ -304,6 +305,10 @@ const USER_INPUT_TOGGLE_TIMING = { export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: ThreadDetailScreenProps) { const navigation = useNavigation(); + const cloudExecution = useThreadCloudExecution({ + environmentId: props.environmentId, + threadId: props.selectedThread.id, + }); const reportedModelSelection = useThreadReportedModelSelection({ environmentId: props.environmentId, threadId: props.selectedThread.id, @@ -1264,6 +1269,26 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread : undefined } > + {cloudExecution ? ( + + + Kilo Cloud · {cloudExecution.repository} · {cloudExecution.branch} + + + Last observation: {cloudExecution.observedAt ?? "unavailable"}. Task:{" "} + {cloudExecution.task}. Sandbox: {cloudExecution.sandbox}. Compute:{" "} + {cloudExecution.billing} + {cloudExecution.billingAttribution === "payer_shared" + ? " (shared account)" + : ""} + . + + + Remote repository. Closing T3 does not stop remote work or billing. Local + files and checkpoints are unavailable. + + + ) : null} {isProviderSubagent ? ( thread.id === selectedThreadDetail.thread.activeProviderThreadId, + )?.nativeMetadata?.cloudExecution || + (routeEnvironmentRuntime?.serverConfig?.providers.some( + (provider) => + provider.instanceId === selectedThread?.providerInstanceId && + provider.driver === "kilo-cloud", + ) ?? + false); const selectedThreadWithDraftSettings = useMemo( () => selectedThread @@ -530,6 +540,7 @@ function ThreadRouteContent( const gitActionProgress = useGitActionProgress(gitActionProgressTarget); const handleOpenGitInspector = useCallback(() => { + if (isCloudThread) return; if (!fileInspector.supported) { if (selectedThread === null) { return; @@ -542,9 +553,16 @@ function ThreadRouteContent( } setInspectorSelection({ routeThreadIdentity, mode: "git" }); showAuxiliaryPane("inspector"); - }, [fileInspector.supported, navigation, routeThreadIdentity, selectedThread, showAuxiliaryPane]); + }, [ + fileInspector.supported, + isCloudThread, + navigation, + routeThreadIdentity, + selectedThread, + showAuxiliaryPane, + ]); const handleOpenFilesInspector = useCallback(() => { - if (selectedThread === null || selectedThreadCwd === null) { + if (isCloudThread || selectedThread === null || selectedThreadCwd === null) { return; } if (!fileInspector.supported) { @@ -560,6 +578,7 @@ function ThreadRouteContent( }); showAuxiliaryPane("inspector"); }, [ + isCloudThread, fileInspector.supported, navigation, props.renderInspector, @@ -664,7 +683,8 @@ function ThreadRouteContent( selectedThreadCwd, ], ); - const activeInspectorRenderer = inspectorMode === null ? undefined : renderInspectorStack; + const activeInspectorRenderer = + isCloudThread || inspectorMode === null ? undefined : renderInspectorStack; // Hand the inspector to the workspace so it renders beside the navigator, // outside this screen's native header — the terminal/git/files toolbar // stays anchored to the chat pane instead of floating above the inspector. @@ -694,7 +714,7 @@ function ThreadRouteContent( hasWorkspaceRoot: Boolean(selectedThreadProject?.workspaceRoot), }); - if (!selectedThread || !selectedThreadProject?.workspaceRoot) { + if (isCloudThread || !selectedThread || !selectedThreadProject?.workspaceRoot) { return; } @@ -704,7 +724,7 @@ function ThreadRouteContent( ...(nextTerminalId ? { terminalId: nextTerminalId } : {}), }); }, - [navigation, selectedThread, selectedThreadProject?.workspaceRoot], + [isCloudThread, navigation, selectedThread, selectedThreadProject?.workspaceRoot], ); const handleOpenNewTerminal = useCallback(() => { @@ -714,7 +734,7 @@ function ThreadRouteContent( listedTerminalIds: terminalMenuSessions.map((session) => session.terminalId), }); - if (!selectedThread || !selectedThreadProject?.workspaceRoot) { + if (isCloudThread || !selectedThread || !selectedThreadProject?.workspaceRoot) { return; } @@ -726,7 +746,13 @@ function ThreadRouteContent( threadId: String(selectedThread.id), terminalId: nextId, }); - }, [navigation, selectedThread, selectedThreadProject?.workspaceRoot, terminalMenuSessions]); + }, [ + isCloudThread, + navigation, + selectedThread, + selectedThreadProject?.workspaceRoot, + terminalMenuSessions, + ]); const handleRunProjectScript = useCallback( async (script: ProjectScript) => { @@ -737,7 +763,7 @@ function ThreadRouteContent( hasWorkspaceRoot: Boolean(selectedThreadProject?.workspaceRoot), }); - if (!selectedThread || !selectedThreadProject?.workspaceRoot) { + if (isCloudThread || !selectedThread || !selectedThreadProject?.workspaceRoot) { terminalDebugLog("project-script:abort", { scriptId: script.id, reason: "no-thread-or-workspace", @@ -790,6 +816,7 @@ function ThreadRouteContent( }); }, [ + isCloudThread, navigation, selectedThread, selectedThreadDetailWorktreePath, @@ -808,8 +835,11 @@ function ThreadRouteContent( } : undefined, onOpenFilesInspector: - fileInspector.supported && selectedThreadCwd !== null ? handleOpenFilesInspector : undefined, - onOpenGitInspector: fileInspector.supported ? handleOpenGitInspector : undefined, + !isCloudThread && fileInspector.supported && selectedThreadCwd !== null + ? handleOpenFilesInspector + : undefined, + onOpenGitInspector: + !isCloudThread && fileInspector.supported ? handleOpenGitInspector : undefined, onMergeBack: mergeBackTargetThreadId !== null && mergeBackRun !== null ? () => void handleMergeBack() @@ -817,8 +847,8 @@ function ThreadRouteContent( currentBranch: selectedThread?.branch ?? null, gitStatus: gitStatus.data, gitOperationLabel: gitState.gitOperationLabel, - canOpenTerminal: Boolean(selectedThreadProject?.workspaceRoot), - canOpenFiles: Boolean(selectedThreadProject?.workspaceRoot), + canOpenTerminal: !isCloudThread && Boolean(selectedThreadProject?.workspaceRoot), + canOpenFiles: !isCloudThread && Boolean(selectedThreadProject?.workspaceRoot), projectScripts: selectedThreadProject ? resolveProjectScripts( routeEnvironmentRuntime?.serverConfig?.settings ?? DEFAULT_SERVER_SETTINGS, @@ -1091,9 +1121,9 @@ function ThreadRouteContent( headerColor={headerColor} usesNativeHeaderGlass={usesNativeHeaderGlass} gitControls={threadGitControlProps} - hasThreadCwd={selectedThreadCwd !== null} - hasWorkspaceRoot={Boolean(selectedThreadProject?.workspaceRoot)} - fileInspectorSupported={fileInspector.supported} + hasThreadCwd={!isCloudThread && selectedThreadCwd !== null} + hasWorkspaceRoot={!isCloudThread && Boolean(selectedThreadProject?.workspaceRoot)} + fileInspectorSupported={!isCloudThread && fileInspector.supported} inspectorMode={inspectorMode} onToggleInspector={handleToggleInspector} onOpenGitInspector={handleOpenGitInspector} diff --git a/apps/mobile/src/state/entities.ts b/apps/mobile/src/state/entities.ts index 3bc9066a399e..2542edd7c6fd 100644 --- a/apps/mobile/src/state/entities.ts +++ b/apps/mobile/src/state/entities.ts @@ -96,3 +96,11 @@ const selectReportedModelSelection = (thread: EnvironmentThread | null) => export function useThreadReportedModelSelection(ref: ScopedThreadRef) { return useAtomValue(environmentThreadDetails.threadAtom(ref), selectReportedModelSelection); } + +const selectCloudExecution = (thread: EnvironmentThread | null) => + thread?.projection.providerThreads.find( + (providerThread) => providerThread.id === thread.projection.thread.activeProviderThreadId, + )?.nativeMetadata?.cloudExecution ?? null; +export function useThreadCloudExecution(ref: ScopedThreadRef) { + return useAtomValue(environmentThreadDetails.threadAtom(ref), selectCloudExecution); +} diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.live.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.live.test.ts new file mode 100644 index 000000000000..61c015ac9b1b --- /dev/null +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.live.test.ts @@ -0,0 +1,459 @@ +// @effect-diagnostics nodeBuiltinImport:off - opt-in live integration with loopback inference for local Kilo. +import * as NodeHttp from "node:http"; +import * as NodeEvents from "node:events"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { describe } from "vite-plus/test"; +import { + OrchestrationV2ThreadProjection, + RunId, + RunAttemptId, + NodeId, + CommandId, + MessageId, + ProjectId, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as Account from "../../provider/kilo/KiloCloudAccount.ts"; +import * as Cloud from "../../provider/kilo/KiloCloudWebClient.ts"; +import * as Journal from "../../provider/kilo/KiloCloudJournal.ts"; +import * as IdAllocator from "../IdAllocator.ts"; +import * as Orchestrator from "../Orchestrator.ts"; +import * as EffectWorker from "../EffectWorker.ts"; +import * as Registry from "../ProviderAdapterRegistry.ts"; +import { makeOrchestratorV2ReplayLayerWithRegistry } from "../testkit/ProviderReplayHarness.ts"; +import * as CloudAdapter from "./KiloCloudAdapterV2.ts"; +import type * as Adapter from "../ProviderAdapter.ts"; +import * as LocalAdapter from "./KiloAdapterV2.ts"; +import * as LocalRuntime from "../../provider/kilo/KiloRuntime.ts"; + +// Explicit paid opt-in, never enabled by CI. A durable one-shot directory prevents +// an accidental rerun from admitting a duplicate sandbox after an uncertain result. +const profile = process.env.KILO_CLOUD_TEST_PROFILE; +const evidence = process.env.KILO_CLOUD_PAID_EVIDENCE; +const repository = process.env.KILO_CLOUD_TEST_REPOSITORY; +const enabled = + process.env.KILO_CLOUD_ALLOW_PAID_TEST === "yes" && + process.env.KILO_CLOUD_ALLOW_FULL_ACCESS_READ_ONLY_TEST === "yes" && + process.env.KILO_BIN && + profile && + evidence && + repository; +const decodeProjection = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadProjection)), +); +const json = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +describe.skipIf(!enabled)("paid Kilo Cloud integration", () => { + it.live( + "reads a synthetic repository and follows up through Orchestrator without local uploads", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(evidence!); // EEXIST deliberately prevents paid retries. + const account = yield* Account.make(profile!); + const credentials = yield* account.load; + const client = Cloud.make({ ...credentials, credentials: account.load }); + const journal = yield* Journal.make(`${evidence}/journal`); + const instanceId = ProviderInstanceId.make("kilo-cloud-paid-test"); + const threadId = ThreadId.make("kilo-cloud-paid-test"); + const modelSelection = { + instanceId, + model: "deepseek/deepseek-v4.1-flash", + options: [{ id: "variant", value: "low" }], + }; + const adapter = yield* CloudAdapter.make({ + instanceId, + continuationKey: "kilo-cloud-paid-test", + accountId: credentials.accountId, + repository: repository!, + branch: "main", + client, + journal, + }); + const localRoot = `${evidence}/local-workspace`; + yield* fs.makeDirectory(localRoot); + yield* fs.writeFileString(`${localRoot}/README.md`, "LOCAL_ONLY_SENTINEL_DO_NOT_UPLOAD"); + const inference = yield* Effect.acquireRelease( + Effect.promise(async () => { + const server = NodeHttp.createServer((request, response) => { + request.resume(); + request.on("end", () => { + response.writeHead(200, { "content-type": "text/event-stream" }); + for (const delta of [{ content: "LOCAL_ONLY_SENTINEL_DO_NOT_UPLOAD" }, {}]) + response.write( + `data: ${JSON.stringify({ id: "local", object: "chat.completion.chunk", created: 0, model: "test", choices: [{ index: 0, delta, finish_reason: Object.keys(delta).length ? null : "stop" }] })}\n\n`, + ); + response.end("data: [DONE]\n\n"); + }); + }); + server.listen(0, "127.0.0.1"); + await NodeEvents.EventEmitter.once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Missing local fixture address"); + return { server, url: `http://127.0.0.1:${address.port}/v1` }; + }), + ({ server }) => + Effect.promise( + () => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + ), + ); + const localInstance = ProviderInstanceId.make("kilo-local-parallel"); + const localThreadId = ThreadId.make("kilo-local-parallel"); + const localModel = { instanceId: localInstance, model: "fixture/test", options: [] }; + const runtime = yield* LocalRuntime.make({ + instanceId: "parallel-local", + binaryPath: process.env.KILO_BIN!, + profileDirectory: `${evidence}/local-profile`, + environment: { + PATH: process.env.PATH, + HTTP_PROXY: process.env.HTTP_PROXY, + HTTPS_PROXY: process.env.HTTPS_PROXY, + NO_PROXY: process.env.NO_PROXY, + NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS, + KILO_DISABLE_AUTOUPDATE: "1", + KILO_DISABLE_MODELS_FETCH: "1", + KILO_DISABLE_DEFAULT_PLUGINS: "1", + KILO_DISABLE_EXTERNAL_SKILLS: "1", + KILO_DISABLE_PROJECT_CONFIG: "1", + KILO_CONFIG_CONTENT: json({ + model: "fixture/test", + small_model: "fixture/test", + plugin: [], + enabled_providers: ["fixture"], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + name: "Local", + options: { baseURL: inference.url }, + models: { test: { name: "Local", limit: { context: 10000, output: 1000 } } }, + }, + }, + }), + }, + }); + const localAdapter = yield* LocalAdapter.make({ + instanceId: localInstance, + continuationKey: "parallel-local", + cwd: localRoot, + runtime, + attachmentsDir: `${evidence}/attachments`, + }); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const terminals = [yield* Deferred.make(), yield* Deferred.make()]; + const seen = new Set(); + const localDone = yield* Deferred.make(); + const timing: Array<{ type: string; threadId: string; status: string; at: number }> = []; + yield* orchestrator.streamStoredEvents.pipe( + Stream.runForEach(({ event }) => + Effect.gen(function* () { + if (event.type === "run.updated") + timing.push({ + type: event.type, + threadId: event.threadId, + status: event.payload.status, + at: yield* Clock.currentTimeMillis, + }); + if ( + event.type === "run.updated" && + event.threadId === localThreadId && + ["completed", "failed", "interrupted"].includes(event.payload.status) + ) + yield* Deferred.succeed(localDone, undefined); + if ( + event.threadId === threadId && + event.type === "run.updated" && + ["completed", "failed", "interrupted"].includes(event.payload.status) && + !seen.has(event.payload.id) + ) { + seen.add(event.payload.id); + const terminal = terminals[seen.size - 1]; + if (terminal) yield* Deferred.succeed(terminal, undefined); + } + }), + ), + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("cloud-live-create"), + createdBy: "user", + creationSource: "web", + threadId, + projectId: ProjectId.make("synthetic"), + title: "Read-only cloud integration", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: `${evidence}/LOCAL-FILES-MUST-NOT-BE-READ`, + }); + const prompts = [ + "Read only README.md and list the top-level file names in this synthetic repository. Reply with a brief summary. Do not execute shell commands, edit files, commit, open a PR, access the network, or start subagents.", + "Using only the context already read, repeat one top-level file name. Do not use tools, modify files, commit, or open a PR. Keep the answer to one line.", + ]; + for (let index = 0; index < prompts.length; index++) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`cloud-live-send-${index}`), + createdBy: "user", + creationSource: "web", + threadId, + messageId: MessageId.make(`cloud-live-user-${index}`), + text: prompts[index]!, + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + }); + if (index === 0) { + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("local-create"), + createdBy: "user", + creationSource: "web", + threadId: localThreadId, + projectId: ProjectId.make("local"), + title: "Parallel local isolation", + modelSelection: localModel, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: localRoot, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("local-send"), + createdBy: "user", + creationSource: "web", + threadId: localThreadId, + messageId: MessageId.make("local-user"), + text: "Say local hello", + attachments: [], + modelSelection: localModel, + dispatchMode: { type: "start_immediately" }, + }); + } + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + yield* Deferred.await(terminals[index]!); + const projection = yield* orchestrator.getThreadProjection(threadId); + yield* fs.writeFileString(`${evidence}/turn-${index}.json`, json(projection)); + assert.equal(projection.runs.at(-1)?.status, "completed"); + assert.equal(projection.checkpoints.length, 0); + assert.isFalse( + projection.messages.some((message) => message.text.includes("LOCAL_ONLY_SENTINEL")), + ); + assert.isFalse(yield* fs.exists(`${evidence}/LOCAL-FILES-MUST-NOT-BE-READ`)); + assert.isTrue( + projection.messages.some( + (message) => message.role === "assistant" && message.text.length > 0, + ), + ); + } + yield* Deferred.await(localDone); + const localProjection = yield* orchestrator.getThreadProjection(localThreadId); + yield* fs.writeFileString( + `${evidence}/parallel-local.json`, + json({ projection: localProjection, timing }), + ); + assert.equal(localProjection.runs.at(-1)?.status, "completed"); + assert.isTrue( + localProjection.messages.some((message) => + message.text.includes("LOCAL_ONLY_SENTINEL"), + ), + ); + assert.isFalse( + localProjection.providerThreads.some((thread) => thread.driver === "kilo-cloud"), + ); + }).pipe( + Effect.provide( + makeOrchestratorV2ReplayLayerWithRegistry( + { + name: "kilo-cloud-paid", + }, + Registry.makeLayer([adapter, localAdapter]), + ), + ), + ); + const entries = yield* journal.read; + const binding = entries.at(-1)?.binding; + if (!binding) + return yield* Effect.die( + new Error("No cloud binding; inspect durable admission before any retry."), + ); + yield* fs.writeFileString( + `${evidence}/lifecycle-after.json`, + json({ + binding, + sandbox: yield* client.sandbox(binding), + billing: yield* client.billing(binding), + }), + ); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 240_000, + ); +}); + +const recovery = process.env.KILO_CLOUD_RECOVER_EVIDENCE; +describe.skipIf(!profile || !recovery)("read-only Kilo Cloud recovery", () => { + it.live( + "reconciles a saved admission through the adapter without resubmitting", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const account = yield* Account.make(profile!); + const credentials = yield* account.load; + const journal = yield* Journal.make(`${recovery}/journal`); + const intent = (yield* journal.read).at(-1); + if (!intent || !intent.binding) + return yield* Effect.die(new Error("Missing saved cloud admission")); + assert.equal(intent.accountId, credentials.accountId); + const client = Cloud.make({ ...credentials, credentials: account.load }); + const adapter = yield* CloudAdapter.make({ + instanceId: intent.providerThread.providerInstanceId, + continuationKey: intent.providerThread.nativeMetadata!.continuationKey!, + accountId: credentials.accountId, + repository: intent.repository, + branch: intent.branch, + client, + journal, + }); + const session = yield* adapter.openSession({ + threadId: intent.providerThread.appThreadId!, + providerSessionId: intent.providerThread.providerSessionId!, + modelSelection: { + instanceId: intent.providerThread.providerInstanceId, + model: "deepseek/deepseek-v4.1-flash", + }, + runtimePolicy: { + runtimeMode: "approval-required", + interactionMode: "default", + cwd: null, + }, + }); + const thread = yield* session.resumeThread({ providerThread: intent.providerThread }); + const snapshot = yield* session.readThreadSnapshot({ providerThread: thread }); + const saved = (yield* journal.read).at(-1)!; + yield* fs.writeFileString( + `${recovery}/recovered.json`, + json({ + state: saved.state, + operationKey: saved.operationKey, + messageId: saved.messageId, + snapshot, + sandbox: yield* client.sandbox(intent.binding), + billing: yield* client.billing(intent.binding), + }), + ); + assert.isTrue(["completed", "failed", "interrupted"].includes(saved.state)); + assert.equal(saved.operationKey, intent.operationKey); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 60_000, + ); +}); + +const controlEvidence = process.env.KILO_CLOUD_CONTROL_EVIDENCE; +describe.skipIf( + !profile || !controlEvidence || process.env.KILO_CLOUD_ALLOW_FULL_ACCESS_READ_ONLY_TEST !== "yes", +)("paid existing-session interrupt", () => { + it.live( + "restores native history and confirms remote interruption in the same cloud worktree", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${controlEvidence}/interrupt-admission`); + const projection = yield* decodeProjection( + yield* fs.readFileString(`${controlEvidence}/turn-1.json`), + ); + const journal = yield* Journal.make(`${controlEvidence}/journal`); + const prior = (yield* journal.read).at(-1)!; + assert.equal(prior.state, "completed"); + const account = yield* Account.make(profile!); + const credentials = yield* account.load; + assert.equal(credentials.accountId, prior.accountId); + const client = Cloud.make({ ...credentials, credentials: account.load }); + const adapter = yield* CloudAdapter.make({ + instanceId: prior.providerThread.providerInstanceId, + continuationKey: prior.providerThread.nativeMetadata!.continuationKey!, + accountId: credentials.accountId, + repository: prior.repository, + branch: prior.branch, + client, + journal, + }); + const runtimePolicy = { + runtimeMode: "full-access" as const, + interactionMode: "default" as const, + cwd: null, + }; + const session = yield* adapter.openSession({ + threadId: projection.thread.id, + providerSessionId: prior.providerThread.providerSessionId!, + modelSelection: projection.thread.modelSelection, + runtimePolicy, + }); + const thread = yield* session.resumeThread({ providerThread: prior.providerThread }); + const restored = yield* session.readThreadSnapshot({ providerThread: thread }); + assert.isTrue(restored.messages.some((message) => message.text === "fixture.py")); + const terminal = yield* Deferred.make(); + yield* session.events.pipe( + Stream.runForEach((event) => + event.type === "turn.terminal" + ? Deferred.succeed(terminal, event).pipe(Effect.asVoid) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* session.startTurn({ + appThread: projection.thread, + threadId: projection.thread.id, + providerThread: thread, + runId: RunId.make("cloud-stop-test"), + runOrdinal: 3, + providerTurnOrdinal: 3, + attemptId: RunAttemptId.make("cloud-stop-test"), + rootNodeId: NodeId.make("cloud-stop-test"), + message: { + messageId: MessageId.make("cloud-stop-test"), + text: "Read-only interruption check: count from one to 200, one number per line. Do not use tools, modify files, commit, open a PR, or start subagents.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: projection.thread.modelSelection, + runtimePolicy, + }); + const admitted = (yield* journal.read).at(-1)!; + assert.equal(admitted.binding?.worktreeId, prior.binding?.worktreeId); + yield* session.interruptTurn({ + providerThread: thread, + providerTurnId: admitted.providerTurn.id, + }); + const event = yield* Deferred.await(terminal); + const result = yield* client.result(admitted.binding!, admitted.messageId); + yield* fs.writeFileString( + `${controlEvidence}/interrupt-result.json`, + json({ + event, + result, + sandbox: yield* client.sandbox(admitted.binding!), + billing: yield* client.billing(admitted.binding!), + }), + ); + assert.equal(result?.status, "interrupted"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 90_000, + ); +}); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts new file mode 100644 index 000000000000..d986ffce105d --- /dev/null +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -0,0 +1,586 @@ +// @effect-diagnostics nodeBuiltinImport:off - external customer API contract over a loopback socket. +import * as NodeHttp from "node:http"; +import * as NodeEvents from "node:events"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { + ProviderSessionId, + ProviderThreadId, + RunId, + RunAttemptId, + CommandId, + MessageId, + ProjectId, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Schema from "effect/Schema"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as Stream from "effect/Stream"; +import * as Cloud from "../../provider/kilo/KiloCloudWebClient.ts"; +import * as Journal from "../../provider/kilo/KiloCloudJournal.ts"; +import * as IdAllocator from "../IdAllocator.ts"; +import * as Orchestrator from "../Orchestrator.ts"; +import * as EffectWorker from "../EffectWorker.ts"; +import * as Registry from "../ProviderAdapterRegistry.ts"; +import { makeOrchestratorV2ReplayLayerWithRegistry } from "../testkit/ProviderReplayHarness.ts"; +import type * as Adapter from "../ProviderAdapter.ts"; +import * as CloudAdapter from "./KiloCloudAdapterV2.ts"; + +const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); +const fixture = Effect.acquireRelease( + Effect.promise(async () => { + let submissions = 0; + let signalInterrupt!: () => void; + const interruptSeen = new Promise((resolve) => { + signalInterrupt = resolve; + }); + const control = { + status: "completed", + missingHistory: false, + incompleteHistory: false, + interruptAccepted: false, + interruptPosts: 0, + permission: false, + answerAccepted: false, + answerPosts: 0, + }; + const conversations = new Map< + string, + { + cloud: string; + native: string; + worktree: string; + initial: string; + messages: Array<{ id: string; prompt: string }>; + } + >(); + const server = NodeHttp.createServer((request, response) => { + let raw = ""; + request.on("data", (chunk) => { + raw += String(chunk); + }); + request.on("end", () => { + const url = new URL(request.url!, "http://localhost"); + const operation = url.pathname.split("/").at(-1)!; + const input = JSON.parse(raw || url.searchParams.get("input") || "{}") as Record< + string, + string + >; + const reply = (data: unknown) => { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ result: { data } })); + }; + if (operation.startsWith("agentProfiles.")) return reply([]); + if (operation === "cloudAgentNext.prepareSession") { + submissions++; + const suffix = String(submissions).padStart(12, "0"); + const state = { + cloud: `workspace_12345678-1234-1234-1234-${suffix}`, + native: `ses_fixture${submissions}`, + worktree: `worktree_12345678-1234-1234-1234-${suffix}`, + initial: input.initialMessageId!, + messages: [{ id: input.initialMessageId!, prompt: input.prompt! }], + }; + conversations.set(state.cloud, state); + return reply({ cloudAgentSessionId: state.cloud, kiloSessionId: state.native }); + } + const state = [...conversations.values()].find( + (item) => item.cloud === input.cloudAgentSessionId || item.native === input.session_id, + ); + if (operation === "cloudAgentNext.interruptSession") { + control.interruptPosts++; + signalInterrupt(); + if (control.interruptAccepted) control.status = "interrupted"; + return reply({ success: control.interruptAccepted }); + } + if (operation === "cloudAgentNext.answerPermission") { + control.answerPosts++; + if (control.answerAccepted) control.permission = false; + return reply({ success: control.answerAccepted }); + } + if (!state) { + response.writeHead(404); + response.end(); + return; + } + if (operation === "cloudAgentNext.getSession") + return reply({ + sessionId: state.cloud, + kiloSessionId: state.native, + worktreeId: state.worktree, + userId: "fixture-account", + githubRepo: "fixture/repo", + upstreamBranch: "main", + autoCommit: false, + initialMessageId: state.initial, + execution: null, + }); + if (operation === "cloudAgentNext.sendMessage") { + const payload = input.payload as unknown as { prompt: string }; + state.messages.push({ id: input.messageId!, prompt: payload.prompt }); + return reply({ + cloudAgentSessionId: state.cloud, + messageId: input.messageId, + status: "started", + delivery: "sent", + }); + } + if (operation === "cloudAgentNext.getPendingInteractions") + return reply({ + permissions: control.permission + ? [ + { + id: "permission-fixture", + sessionID: state.native, + permission: "read", + patterns: ["README.md"], + }, + ] + : [], + questions: [], + }); + if (operation === "cloudAgentNext.getMessageResult") + return reply({ + cloudAgentSessionId: state.cloud, + messageId: input.messageId, + status: control.status, + }); + if (operation === "cliSessionsV2.getSessionMessagesPage" && control.missingHistory) + return reply({ kiloSessionId: state.native, history: null, watermarkEventId: 49 }); + if (operation === "cliSessionsV2.getSessionMessagesPage") + return reply({ + kiloSessionId: state.native, + watermarkEventId: 3, + history: { + nextCursor: null, + omittedItemCount: 0, + messages: state.messages.flatMap((message) => [ + { + info: { + id: message.id, + sessionID: state.native, + role: "user", + time: { created: 1 }, + }, + parts: [ + { + id: `part-${message.id}`, + messageID: message.id, + sessionID: state.native, + type: "text", + text: message.prompt, + }, + ], + }, + { + info: { + id: `reply-${message.id}`, + sessionID: state.native, + role: "assistant", + finish: "stop", + parentID: message.id, + time: + control.incompleteHistory && message !== state.messages[0] + ? { created: 2 } + : { created: 2, completed: 3 }, + }, + parts: [ + { + id: `part-reply-${message.id}`, + messageID: `reply-${message.id}`, + sessionID: state.native, + type: "text", + text: `Remote reply: ${message.prompt}`, + }, + ], + }, + ]), + }, + }); + response.writeHead(400); + response.end(); + }); + }); + server.listen(0, "127.0.0.1"); + await NodeEvents.EventEmitter.once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("No fixture address"); + return { + origin: `http://127.0.0.1:${address.port}`, + submissions: () => submissions, + conversations, + control, + interruptSeen, + close: async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }, + }; + }), + (fixture) => Effect.promise(fixture.close), +); + +it.live( + "completes two isolated cloud threads through SQLite orchestration without touching local workspaces", + () => + Effect.gen(function* () { + const remote = yield* fixture; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + const journal = yield* Journal.make(directory); + const instanceId = ProviderInstanceId.make("cloud-test"); + const modelSelection = { instanceId, model: "fixture/model" }; + let restore: Adapter.ProviderAdapterV2TurnInput | undefined; + const adapterOptions = { + instanceId, + continuationKey: "fixture-account-repo", + accountId: "fixture-account", + repository: "fixture/repo", + branch: "main", + client: Cloud.make({ + accountId: "fixture-account", + token: Redacted.make("fixture-token"), + origin: remote.origin, + }), + journal, + }; + const adapter = yield* CloudAdapter.make(adapterOptions); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const done = yield* Deferred.make(); + const completed = new Set(); + yield* orchestrator.streamStoredEvents.pipe( + Stream.runForEach(({ event }) => { + if ( + event.type === "run.updated" && + ["completed", "failed", "interrupted"].includes(event.payload.status) + ) + completed.add(event.threadId); + return completed.size === 2 + ? Deferred.succeed(done, undefined).pipe(Effect.asVoid) + : Effect.void; + }), + Effect.forkScoped, + ); + for (const suffix of ["a", "b"]) { + const threadId = ThreadId.make(`cloud-${suffix}`); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`create-${suffix}`), + createdBy: "user", + creationSource: "web", + threadId, + projectId: ProjectId.make("cloud-project"), + title: "Cloud contract", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: `${directory}/must-not-exist-${suffix}`, + }); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`send-${suffix}`), + createdBy: "user", + creationSource: "web", + threadId, + messageId: MessageId.make(`user-${suffix}`), + text: `isolation-${suffix}`, + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + }); + } + yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); + yield* Deferred.await(done); + for (const suffix of ["a", "b"]) { + const projection = yield* orchestrator.getThreadProjection( + ThreadId.make(`cloud-${suffix}`), + ); + assert.equal( + projection.runs[0]?.status, + "completed", + yield* encodeJson(projection.turnItems), + ); + assert.isNotNull(projection.runs[0]?.completedAt); + assert.isTrue( + projection.messages.some( + (message) => + message.role === "assistant" && + message.text === `Remote reply: isolation-${suffix}`, + ), + ); + assert.isFalse( + projection.messages.some((message) => + message.text.includes(`isolation-${suffix === "a" ? "b" : "a"}`), + ), + ); + assert.isFalse(yield* fs.exists(`${directory}/must-not-exist-${suffix}`)); + assert.equal(projection.checkpoints.length, 0); + if (suffix === "a") { + const run = projection.runs[0]; + const providerThread = projection.providerThreads[0]; + if (!run?.rootNodeId || !run.activeAttemptId || !providerThread) + return yield* Effect.die(new Error("Missing persisted turn")); + restore = { + appThread: projection.thread, + threadId: projection.thread.id, + runId: run.id, + runOrdinal: run.ordinal, + providerTurnOrdinal: 1, + attemptId: run.activeAttemptId, + rootNodeId: run.rootNodeId, + providerThread, + message: { + messageId: MessageId.make("user-a"), + text: "MUST NOT BE RESUBMITTED", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: null, + }, + reattach: true, + }; + } + } + }).pipe( + Effect.provide( + makeOrchestratorV2ReplayLayerWithRegistry( + { name: "kilo-cloud-contract" }, + Registry.makeSingleLayer({ + ...adapter, + openSession: (input) => + adapter.openSession(input).pipe( + Effect.map((runtime) => ({ + ...runtime, + startTurn: (input) => + runtime + .startTurn(input) + .pipe( + Effect.catchCause((cause) => + Effect.logError(Cause.pretty(cause)).pipe( + Effect.andThen(Effect.failCause(cause)), + ), + ), + ), + })), + ), + }), + ), + ), + ); + assert.equal(remote.submissions(), 2); + const entries = yield* journal.read; + assert.equal(entries.length, 2); + assert.equal(new Set(entries.map((entry) => entry.binding?.worktreeId)).size, 2); + assert.isTrue(entries.every((entry) => entry.state === "completed")); + if (!restore) return yield* Effect.die(new Error("Missing restore input")); + // Simulate loss of the terminal T3 event after the durable journal commit. + // Reattaching a fresh runtime must replay terminality without another paid POST. + const restored = yield* adapter.openSession({ + threadId: restore.threadId, + providerSessionId: ProviderSessionId.make("cloud-restored"), + modelSelection, + runtimePolicy: restore.runtimePolicy, + }); + const restoredThread = yield* restored.ensureThread({ + threadId: restore.threadId, + existingProviderThread: restore.providerThread, + modelSelection, + runtimePolicy: restore.runtimePolicy, + }); + const terminal = yield* Deferred.make(); + const replayedMessages: string[] = []; + const restoredEvents = yield* restored.events.pipe( + Stream.runForEach((event) => + Effect.gen(function* () { + if (event.type === "message.updated") replayedMessages.push(event.message.text); + if (event.type === "turn.terminal") yield* Deferred.succeed(terminal, event); + }), + ), + Effect.forkScoped, + ); + yield* restored.startTurn({ ...restore, providerThread: restoredThread }); + const event = yield* Deferred.await(terminal); + assert.isTrue(event.type === "turn.terminal" && event.status === "completed"); + assert.include(replayedMessages, "Remote reply: isolation-a"); + assert.equal(remote.submissions(), 2); + const restricted = yield* restored + .startTurn({ + ...restore, + reattach: false, + providerThread: restoredThread, + runtimePolicy: { ...restore.runtimePolicy, runtimeMode: "approval-required" }, + }) + .pipe(Effect.flip); + assert.include(restricted.message, "cannot enforce restricted permissions"); + assert.equal(remote.submissions(), 2); + assert.equal((yield* journal.read).length, 2); + yield* Fiber.interrupt(restoredEvents); + remote.control.status = "running"; + remote.control.permission = true; + remote.control.incompleteHistory = true; + const pendingRequest = + yield* Deferred.make< + Extract + >(); + const resolvedItems: Array = []; + const failedWithoutHistory = yield* Deferred.make(); + yield* restored.events.pipe( + Stream.runForEach((event) => + Effect.gen(function* () { + resolvedItems.push(event); + if (event.type === "turn.terminal" && event.status === "failed") + yield* Deferred.succeed(failedWithoutHistory, undefined); + if ( + event.type === "runtime_request.updated" && + event.runtimeRequest.status === "pending" + ) + yield* Deferred.succeed(pendingRequest, event); + }), + ), + Effect.forkScoped, + ); + yield* restored.startTurn({ + ...restore, + reattach: false, + providerThread: restoredThread, + runId: RunId.make("followup-run"), + attemptId: RunAttemptId.make("followup-attempt"), + runOrdinal: 2, + providerTurnOrdinal: 2, + message: { + ...restore.message, + messageId: MessageId.make("followup"), + text: "Follow up in the same workspace", + }, + }); + const pending = yield* Deferred.await(pendingRequest); + yield* restored + .respondToRuntimeRequest({ requestId: pending.runtimeRequest.id, decision: "accept" }) + .pipe(Effect.flip); + remote.control.answerAccepted = true; + yield* restored.respondToRuntimeRequest({ + requestId: pending.runtimeRequest.id, + decision: "accept", + }); + assert.equal(remote.control.answerPosts, 2); + const currentTurn = (yield* journal.read).at(-1)!; + // A definite rejection permits an explicit retry. It is never an automatic resend. + const rejectedStop = yield* restored + .interruptTurn({ + providerThread: restoredThread, + providerTurnId: currentTurn.providerTurn.id, + }) + .pipe(Effect.forkScoped); + yield* Effect.promise(() => remote.interruptSeen); + remote.control.interruptAccepted = true; + yield* restored.interruptTurn({ + providerThread: restoredThread, + providerTurnId: currentTurn.providerTurn.id, + }); + yield* Fiber.join(rejectedStop); + assert.equal(remote.control.interruptPosts, 2); + assert.equal(remote.submissions(), 2); + assert.isTrue( + resolvedItems.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "approval_request" && + event.turnItem.status === "completed", + ), + ); + // Reopening with paid admission disabled still restores control and native + // history; incomplete records from an interrupted turn must stay terminal. + const controlOnly = yield* CloudAdapter.make({ ...adapterOptions, allowAdmission: false }); + const reopened = yield* controlOnly.openSession({ + threadId: restore.threadId, + providerSessionId: ProviderSessionId.make("cloud-control-only"), + modelSelection, + runtimePolicy: restore.runtimePolicy, + }); + const reopenedThread = yield* reopened.resumeThread({ providerThread: restoredThread }); + const snapshot = yield* reopened.readThreadSnapshot({ providerThread: reopenedThread }); + assert.isTrue( + snapshot.messages.some( + (message) => message.text === "Remote reply: Follow up in the same workspace", + ), + ); + assert.isTrue(snapshot.messages.every((message) => !message.streaming)); + const denied = yield* reopened + .startTurn({ ...restore, providerThread: reopenedThread, reattach: false }) + .pipe(Effect.flip); + assert.include(denied.message, "Paid cloud execution is disabled"); + assert.equal(remote.submissions(), 2); + remote.control.status = "failed"; + remote.control.missingHistory = true; + yield* restored.startTurn({ + ...restore, + reattach: false, + providerThread: restoredThread, + runId: RunId.make("failed-run"), + attemptId: RunAttemptId.make("failed-attempt"), + runOrdinal: 3, + providerTurnOrdinal: 3, + message: { + ...restore.message, + messageId: MessageId.make("failed"), + text: "Bootstrap failure has no history", + }, + }); + yield* Deferred.await(failedWithoutHistory); + assert.equal((yield* journal.read).at(-1)?.state, "failed"); + assert.equal(remote.submissions(), 2); + const first = (yield* journal.read)[0]!; + yield* journal.save({ ...first, interruptRequested: true }); + assert.equal((yield* replacementForStale()).operation, "write"); + function replacementForStale() { + return journal.save({ ...first, interruptRequested: false }).pipe(Effect.flip); + } + assert.equal( + (yield* journal.save({ ...first, accountId: "another-account" }).pipe(Effect.flip)) + .operation, + "write", + ); + assert.equal( + (yield* journal.save({ ...first, state: "active" }).pipe(Effect.flip)).operation, + "write", + ); + const replacement = yield* Journal.make(directory); + const concurrent = { + ...first, + state: "admission_unknown" as const, + providerThread: { + ...first.providerThread, + id: ProviderThreadId.make("same-racing-thread"), + }, + }; + const reservations = yield* Effect.all( + [ + journal.reserve({ ...concurrent, operationKey: "race-a" }), + replacement.reserve({ ...concurrent, operationKey: "race-b" }), + ], + { concurrency: "unbounded" }, + ); + assert.equal(reservations.filter(Boolean).length, 1); + assert.equal( + (yield* replacement.read).filter( + (entry) => entry.providerThread.id === concurrent.providerThread.id, + ).length, + 1, + ); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 30_000, +); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts new file mode 100644 index 000000000000..03f0540c0198 --- /dev/null +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -0,0 +1,1247 @@ +import { + ProviderDriverKind, + RuntimeRequestId, + type OrchestrationV2ProviderCapabilities, + type OrchestrationV2ProviderThread, + type OrchestrationV2ConversationMessage, + type OrchestrationV2TurnItem, + type OrchestrationV2RuntimeRequest, + type OrchestrationV2ExecutionNode, + type ProviderInstanceId, +} from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Crypto from "effect/Crypto"; +import * as Deferred from "effect/Deferred"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Encoding from "effect/Encoding"; +import * as Queue from "effect/Queue"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; +import * as Cloud from "../../provider/kilo/KiloCloudWebClient.ts"; +import { KiloCloudError } from "../../provider/kilo/KiloCloudClient.ts"; +import * as Journal from "../../provider/kilo/KiloCloudJournal.ts"; +import * as IdAllocator from "../IdAllocator.ts"; +import * as Adapter from "../ProviderAdapter.ts"; +import { turnScopedSelectionTransition } from "../ProviderSelectionTransition.ts"; +import { makeProviderFailure } from "../ProviderFailure.ts"; +import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; +import { openCodePermissionRules } from "./OpenCodeAdapterV2.ts"; +import { openCodeToolTurnItem } from "./OpenCodeToolItems.ts"; + +export const KILO_CLOUD_PROVIDER = ProviderDriverKind.make("kilo-cloud"); +const capabilities: OrchestrationV2ProviderCapabilities = { + sessions: { + supportsMultipleProviderThreadsPerSession: false, + supportsModelSwitchInSession: true, + supportsProviderSwitchingViaHandoff: false, + supportsRuntimeModeSwitchInSession: false, + pendingRequestsSurviveRestart: true, + }, + threads: { + canCreateEmptyThread: true, + canReadThreadSnapshot: true, + canRollbackThread: false, + canForkThread: false, + canForkFromTurn: false, + canForkFromSubagentThread: false, + exposesNativeThreadId: true, + }, + turns: { + exposesNativeTurnId: false, + emitsTurnStarted: true, + emitsTurnCompleted: true, + supportsInterrupt: true, + supportsActiveSteering: false, + supportsSteeringByInterruptRestart: false, + supportsQueuedMessages: false, + terminalStatusQuality: "strong", + }, + streaming: { + streamsAssistantText: false, + streamsReasoning: false, + streamsToolOutput: false, + streamsPlanText: false, + emitsMessageCompleted: true, + }, + tools: { + exposesToolItemIds: true, + emitsToolStarted: true, + emitsToolCompleted: true, + emitsToolOutput: true, + supportsMcpTools: false, + supportsDynamicToolCallbacks: false, + }, + approvals: { + supportsCommandApproval: false, + supportsFileReadApproval: false, + supportsFileChangeApproval: false, + supportsApplyPatchApproval: false, + approvalsHaveNativeRequestIds: true, + approvalCallbacksAreLiveOnly: true, + approvalsCanOriginateFromSubagents: false, + }, + planning: { + emitsPlanUpdated: false, + emitsTodoList: false, + emitsProposedPlan: false, + supportsStructuredQuestions: true, + planDeltasHaveItemIds: false, + }, + subagents: { + supportsSubagents: false, + exposesSubagentThreadIds: false, + emitsSubagentLifecycle: false, + canWaitForSubagents: false, + canCloseSubagents: false, + canForkSubagentThread: false, + }, + context: { + acceptsSystemContext: false, + acceptsDeveloperContext: false, + acceptsSyntheticUserContext: false, + canGenerateSummaries: false, + canConsumeHandoffSummaries: false, + supportsDeltaHandoff: false, + supportsFullThreadHandoff: false, + maxRecommendedHandoffChars: null, + }, + checkpointing: { + appCanCheckpointFilesystem: false, + supportsNestedCheckpointScopes: false, + providerCanRollbackConversation: false, + providerRollbackReturnsSnapshot: false, + providerCanReadConversationSnapshot: true, + }, + identity: { + nativeThreadIds: "strong", + nativeTurnIds: "weak", + nativeItemIds: "strong", + nativeRequestIds: "strong", + }, + runtimePolicy: { enforcement: "client-boundary" }, +}; + +const error = (detail: string) => + new Adapter.ProviderAdapterProtocolError({ driver: KILO_CLOUD_PROVIDER, detail }); +const nativeRef = (nativeId: string) => ({ + driver: KILO_CLOUD_PROVIDER, + nativeId, + strength: "strong" as const, +}); +const wire = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError( + (cause) => + new Adapter.ProviderAdapterProtocolError({ + driver: KILO_CLOUD_PROVIDER, + detail: "Kilo Cloud request failed. Remote execution and billing may still be active.", + cause, + }), + ), + ); +const encode = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const isCloudError = Schema.is(KiloCloudError); +const isRecord = Schema.is(Schema.Record(Schema.String, Schema.Unknown)); + +export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { + readonly instanceId: ProviderInstanceId; + readonly continuationKey: string; + readonly accountId: string; + readonly repository: string; + readonly branch: string; + readonly client: ReturnType; + readonly journal: Effect.Success>; + readonly allowAdmission?: boolean; +}) { + const ids = yield* IdAllocator.IdAllocatorV2; + const crypto = yield* Crypto.Crypto; + const driver = KILO_CLOUD_PROVIDER; + const key = (id: string) => `${options.continuationKey}:${id}`; + return Adapter.ProviderAdapterV2.of({ + instanceId: options.instanceId, + driver, + getCapabilities: () => Effect.succeed(capabilities), + planSelectionTransition: () => Effect.succeed(turnScopedSelectionTransition()), + openSession: Effect.fn("KiloCloudAdapterV2.openSession")(function* (input) { + const scope = yield* Effect.scope; + const gate = yield* Semaphore.make(1); + const now = yield* DateTime.now; + const session = { + id: input.providerSessionId, + driver, + providerInstanceId: options.instanceId, + cwd: `kilo-cloud://${options.repository}`, + model: input.modelSelection.model, + status: "ready" as const, + capabilities, + createdAt: now, + updatedAt: now, + lastError: null, + }; + const queue = yield* Queue.unbounded(); + const wake = yield* Queue.sliding(1); + const nodes = new Map(); + const items = new Map(); + const emit = (event: Adapter.ProviderAdapterV2Event) => + Effect.suspend(() => { + if (event.type === "node.updated") nodes.set(event.node.id, event.node); + if (event.type === "turn_item.updated") items.set(event.turnItem.id, event.turnItem); + return Queue.offer(queue, event).pipe(Effect.asVoid); + }); + let thread: OrchestrationV2ProviderThread | undefined; + let active: Journal.CloudIntent | undefined; + let needsHistoryRestore = false; + let binding: Cloud.CloudBinding | undefined; + let watching = false; + let streamWatching = false; + let streamCursor = 0; + let monitorSandbox = false; + let taskState: + | "not_started" + | "admission_unknown" + | "queued" + | "running" + | "completed" + | "failed" + | "interrupted" + | "unknown" = "not_started"; + let lifecycleSignature = ""; + let terminalSignal = yield* Deferred.make(); + const messages = new Map(); + const signatures = new Map(); + const requests = new Map< + RuntimeRequestId, + { + runtime: OrchestrationV2RuntimeRequest; + native: Cloud.CloudInteraction; + node: OrchestrationV2ExecutionNode; + item: OrchestrationV2TurnItem; + } + >(); + const ordinals = new Map(); + const ordinal = (id: string) => { + const old = ordinals.get(id); + if (old !== undefined) return old; + const next = ordinals.size + 1; + ordinals.set(id, next); + return next; + }; + const status = (detail: string) => + Effect.gen(function* () { + yield* emit({ + type: "provider_session.updated", + driver, + providerSession: { + ...session, + status: "waiting", + lastError: detail, + updatedAt: yield* DateTime.now, + }, + }); + }); + const owned = (candidate: OrchestrationV2ProviderThread) => + candidate.id === thread?.id && + candidate.providerInstanceId === options.instanceId && + candidate.nativeMetadata?.continuationKey === options.continuationKey + ? Effect.void + : Effect.fail(error("This cloud thread belongs to a different account or repository.")); + const save = (intent: Journal.CloudIntent) => + wire(options.journal.save(intent)).pipe( + Effect.tap((saved) => + Effect.sync(() => { + active = saved; + }), + ), + ); + const resolveRequest = Effect.fn("KiloCloudAdapterV2.resolveRequest")(function* ( + entry: NonNullable>, + cancelled = false, + ) { + const at = yield* DateTime.now; + entry.runtime = { + ...entry.runtime, + status: cancelled ? "cancelled" : "resolved", + resolvedAt: at, + }; + const status = cancelled ? ("interrupted" as const) : ("completed" as const); + entry.node = { ...entry.node, status, completedAt: at }; + entry.item = { ...entry.item, status, completedAt: at, updatedAt: at }; + yield* emit({ type: "runtime_request.updated", driver, runtimeRequest: entry.runtime }); + yield* emit({ type: "node.updated", driver, node: entry.node }); + yield* emit({ type: "turn_item.updated", driver, turnItem: entry.item }); + }); + const finish = Effect.fn("KiloCloudAdapterV2.finish")(function* ( + terminal: "completed" | "failed" | "interrupted", + ) { + if (!active) return; + const at = yield* DateTime.now; + const saved = { + ...active, + state: terminal, + providerTurn: { ...active.providerTurn, status: terminal, completedAt: at }, + }; + yield* save(saved); + if (thread?.nativeMetadata?.cloudExecution) { + thread = { + ...thread, + status: "idle", + nativeMetadata: { + ...thread.nativeMetadata, + cloudExecution: { ...thread.nativeMetadata.cloudExecution, task: terminal }, + }, + }; + yield* emit({ type: "provider_thread.updated", driver, providerThread: thread }); + } + yield* emit({ type: "provider_turn.updated", driver, providerTurn: saved.providerTurn }); + for (const request of requests.values()) { + if (request.runtime.status !== "pending") continue; + yield* resolveRequest(request, true); + } + for (const node of nodes.values()) { + if ( + node.providerTurnId === saved.providerTurn.id && + ["running", "waiting"].includes(node.status) + ) + yield* emit({ + type: "node.updated", + driver, + node: { ...node, status: terminal, completedAt: at }, + }); + } + for (const item of items.values()) { + if ( + item.providerTurnId !== saved.providerTurn.id || + !["running", "waiting"].includes(item.status) + ) + continue; + const done = { ...item, status: terminal, completedAt: at, updatedAt: at }; + yield* emit({ + type: "turn_item.updated", + driver, + turnItem: "streaming" in done ? { ...done, streaming: false } : done, + }); + } + for (const [id, message] of messages) { + if (!message.streaming) continue; + const done = { ...message, streaming: false, updatedAt: at }; + messages.set(id, done); + yield* emit({ type: "message.updated", driver, message: done }); + } + yield* emit( + terminal === "failed" + ? { + type: "turn.terminal", + driver, + providerThreadId: saved.providerThread.id, + providerTurnId: saved.providerTurn.id, + runOrdinal: saved.runOrdinal, + failureItemOrdinal: ordinals.size + 1, + status: "failed", + failure: makeProviderFailure({ + class: "provider_error", + code: "provider_error", + message: "Kilo Cloud reported a failed task.", + }), + threadDisposition: "reusable", + } + : { + type: "turn.terminal", + driver, + providerThreadId: saved.providerThread.id, + providerTurnId: saved.providerTurn.id, + runOrdinal: saved.runOrdinal, + status: terminal, + failure: null, + threadDisposition: "reusable", + }, + ); + yield* emit({ + type: "provider_session.updated", + driver, + providerSession: { ...session, status: "ready", updatedAt: at, lastError: null }, + }); + active = undefined; + if (!binding) monitorSandbox = false; + taskState = terminal; + yield* Deferred.succeed(terminalSignal, undefined); + }); + const project = Effect.fn("KiloCloudAdapterV2.project")(function* ( + message: Cloud.CloudMessage, + intents: ReadonlyArray, + ) { + if (!thread?.appThreadId) return; + const correlationId = + message.info.role === "user" ? message.info.id : message.info.parentID; + const intent = intents.find((entry) => entry.messageId === correlationId); + if (!intent) return; // No unrelated native conversation or child events enter this T3 thread. + const correlation = + intent.providerThread.nativeMetadata?.turnCorrelations?.[intent.messageId]; + if (!correlation) return; + const terminal = ["completed", "failed", "interrupted"].includes(intent.state); + const signature = encode([message, intent.state]); + if (signatures.get(message.info.id) === signature) return; + const at = yield* DateTime.now; + const done = + terminal || message.info.role === "user" || message.info.time.completed !== undefined; + const text = message.parts + .filter((p) => p.type === "text") + .map((p) => p.text ?? "") + .join(""); + const row: OrchestrationV2ConversationMessage = { + id: + message.info.role === "user" + ? correlation.messageId + : ids.derive.messageFromProviderItem({ driver, nativeItemId: key(message.info.id) }), + threadId: thread.appThreadId, + runId: correlation.runId, + nodeId: correlation.nodeId, + role: message.info.role, + text, + attachments: [], + streaming: !done, + createdAt: DateTime.makeUnsafe(message.info.time.created), + updatedAt: at, + createdBy: message.info.role === "user" ? (correlation.createdBy ?? "user") : "agent", + creationSource: + message.info.role === "user" ? (correlation.creationSource ?? "provider") : "provider", + }; + messages.set(message.info.id, row); + yield* emit({ type: "message.updated", driver, message: row }); + if (message.info.role !== "assistant") { + signatures.set(message.info.id, signature); + return; + } + for (const part of message.parts) { + if (part.type !== "text" && part.type !== "reasoning" && part.type !== "tool") continue; + const nodeId = ids.derive.nodeFromProviderItem({ driver, nativeItemId: key(part.id) }); + const toolStatus = part.state?.status; + const nativeDone = + part.type === "tool" + ? toolStatus === "completed" || toolStatus === "error" + : message.info.time.completed !== undefined; + const partDone = nativeDone || terminal; + const partStatus = + toolStatus === "error" + ? "failed" + : nativeDone + ? "completed" + : terminal + ? intent.state + : "running"; + const base = { + id: ids.derive.turnItemFromProviderItem({ driver, nativeItemId: key(part.id) }), + threadId: thread.appThreadId, + runId: correlation.runId, + nodeId, + providerThreadId: thread.id, + providerTurnId: intent.providerTurn.id, + nativeItemRef: nativeRef(part.id), + parentItemId: null, + ordinal: ordinal(part.id), + status: partStatus as "completed" | "failed" | "interrupted" | "running", + title: part.tool ?? null, + startedAt: intent.providerTurn.startedAt, + completedAt: partDone ? at : null, + updatedAt: at, + }; + yield* emit({ + type: "node.updated", + driver, + node: { + id: nodeId, + threadId: thread.appThreadId, + runId: correlation.runId, + parentNodeId: correlation.nodeId, + rootNodeId: correlation.nodeId, + kind: + part.type === "tool" + ? "tool_call" + : part.type === "reasoning" + ? "reasoning" + : "assistant_message", + status: base.status, + countsForRun: false, + providerThreadId: thread.id, + providerTurnId: intent.providerTurn.id, + nativeItemRef: base.nativeItemRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: base.startedAt, + completedAt: base.completedAt, + }, + }); + const item: OrchestrationV2TurnItem = + part.type === "tool" + ? openCodeToolTurnItem(base, { + name: part.tool ?? "unknown", + input: isRecord(part.state?.input) ? part.state.input : {}, + output: + typeof part.state?.output === "string" + ? part.state.output + : typeof part.state?.error === "string" + ? part.state.error + : undefined, + completedMetadata: toolStatus === "completed" ? part.state?.metadata : undefined, + }) + : part.type === "reasoning" + ? { ...base, type: "reasoning", text: part.text ?? "", streaming: !done } + : { + ...base, + type: "assistant_message", + messageId: row.id, + text: part.text ?? "", + streaming: !done, + }; + yield* emit({ type: "turn_item.updated", driver, turnItem: item }); + } + signatures.set(message.info.id, signature); + }); + const ask = Effect.fn("KiloCloudAdapterV2.ask")(function* (native: Cloud.CloudInteraction) { + const intent = active; + const correlation = + intent?.providerThread.nativeMetadata?.turnCorrelations?.[intent.messageId]; + if ( + !intent || + !correlation || + !thread?.appThreadId || + requests.has(RuntimeRequestId.make(key(native.id))) + ) + return; + const running = { + turn: intent.providerTurn, + input: { + threadId: thread.appThreadId, + runId: correlation.runId, + rootNodeId: correlation.nodeId, + }, + }; + const at = yield* DateTime.now; + const requestId = RuntimeRequestId.make(key(native.id)); + const nodeId = ids.derive.approvalNode({ requestId }); + const question = "questions" in native; + const kind = question + ? "user_input" + : /edit|write|patch/.test(native.permission) + ? "file-change" + : /read|glob|grep/.test(native.permission) + ? "file-read" + : "command"; + const runtime: OrchestrationV2RuntimeRequest = { + id: requestId, + nodeId, + providerTurnId: running.turn.id, + nativeRequestRef: nativeRef(native.id), + kind, + status: "pending", + responseCapability: { type: "live", providerSessionId: input.providerSessionId }, + createdAt: at, + resolvedAt: null, + }; + const node: OrchestrationV2ExecutionNode = { + id: nodeId, + threadId: running.input.threadId, + runId: running.input.runId, + parentNodeId: running.input.rootNodeId, + rootNodeId: running.input.rootNodeId, + kind: question ? "user_input_request" : "approval_request", + status: "waiting", + countsForRun: false, + providerThreadId: thread.id, + providerTurnId: running.turn.id, + nativeItemRef: nativeRef(native.id), + runtimeRequestId: requestId, + checkpointScopeId: null, + startedAt: at, + completedAt: null, + }; + yield* emit({ type: "node.updated", driver, node }); + yield* emit({ + type: "runtime_request.updated", + driver: driver, + threadId: running.input.threadId, + runtimeRequest: runtime, + }); + const base = { + id: ids.derive.approvalTurnItem({ requestId }), + threadId: running.input.threadId, + runId: running.input.runId, + nodeId, + providerThreadId: thread.id, + providerTurnId: running.turn.id, + nativeItemRef: nativeRef(native.id), + parentItemId: null, + ordinal: ordinal(native.id), + status: "waiting" as const, + startedAt: at, + completedAt: null, + updatedAt: at, + requestId, + }; + const turnItem: OrchestrationV2TurnItem = question + ? { + ...base, + type: "user_input_request", + title: "Kilo question", + questions: native.questions.map((q, index) => ({ + id: String(index), + header: q.header, + question: q.question, + options: q.options, + multiSelect: q.multiple ?? false, + allowCustomAnswer: q.custom ?? true, + })), + } + : { + ...base, + type: "approval_request", + title: native.permission, + requestKind: kind === "user_input" ? "command" : kind, + prompt: native.patterns.join("\n"), + }; + requests.set(requestId, { runtime, native, node, item: turnItem }); + yield* emit({ type: "turn_item.updated", driver: driver, turnItem }); + }); + const reconcile = Effect.fn("KiloCloudAdapterV2.reconcile")(function* () { + const expectedMessageId = active?.messageId; + if (!binding) { + if (active && !active.prepared) { + const found = yield* wire( + options.client.findAdmission(options.repository, active.messageId), + ); + if (found && active?.messageId === expectedMessageId) + yield* save({ ...active, prepared: found }); + } + if (active?.prepared) { + binding = yield* wire( + options.client.bind( + active.prepared, + options.repository, + active.messageId, + options.branch, + ), + ); + yield* save({ ...active, binding, state: "active" }); + } else return; + } + const intents = yield* wire(options.journal.readThread(thread!.id)); + const persisted = intents.find((entry) => entry.messageId === expectedMessageId); + if ( + persisted && + active && + active.messageId === expectedMessageId && + persisted.revision > active.revision + ) + active = persisted; + const outcome = active + ? yield* wire(options.client.result(binding, active.messageId)) + : null; + if (active?.messageId !== expectedMessageId) return; + if (outcome) taskState = outcome.status; + const failedOrInterrupted = + outcome?.status === "failed" || outcome?.status === "interrupted"; + let cursor: string | undefined; + let finalReplySeen = false; + const seen = new Set(); + const readHistory = Effect.gen(function* () { + do { + const page = yield* wire(options.client.history(binding!, cursor)); + if (page.history === null) break; + if (page.history.omittedItemCount > 0) + return yield* error( + "Kilo Cloud history is incomplete; remote task state is still unknown.", + ); + for (const message of page.history.messages) { + yield* project(message, intents); + if ( + message.info.parentID === expectedMessageId && + message.info.time.completed !== undefined && + message.info.finish && + message.info.finish !== "tool-calls" + ) + finalReplySeen = true; + } + cursor = + active && + page.history.messages.some((message) => message.info.id === expectedMessageId) + ? undefined + : (page.history.nextCursor ?? undefined); + if (cursor && seen.has(cursor)) + return yield* error("Kilo Cloud returned a repeated history cursor."); + if (cursor) seen.add(cursor); + } while (cursor); + needsHistoryRestore = false; + }); + if (failedOrInterrupted) { + needsHistoryRestore = true; + // Confirmed termination survives unavailable bootstrap history, but retain any + // output produced while this client was disconnected before closing the turn. + yield* readHistory.pipe(Effect.timeout("5 seconds"), Effect.ignore); + yield* finish(outcome.status as "failed" | "interrupted").pipe(Effect.uninterruptible); + return; + } + yield* readHistory; + if (active && active.messageId === expectedMessageId) { + const pending = yield* wire(options.client.pending(binding)); + if (active?.messageId !== expectedMessageId) return; + for (const interaction of [...pending.questions, ...pending.permissions]) + yield* ask(interaction); + const stillPending = new Set( + [...pending.questions, ...pending.permissions].map((request) => request.id), + ); + for (const entry of requests.values()) { + if (entry.runtime.status === "pending" && !stillPending.has(entry.native.id)) { + yield* resolveRequest(entry); + } + } + const result = outcome; + if (active?.messageId !== expectedMessageId) return; + if (result) taskState = result.status; + if ( + result && + result.status !== "queued" && + result.status !== "running" && + (result.status !== "completed" || finalReplySeen) + ) + yield* finish(result.status).pipe(Effect.uninterruptible); + } + }); + const lifecycle = Effect.gen(function* () { + if (!thread || !binding) return; + const sandbox = yield* options.client.sandbox(binding).pipe( + Effect.timeout("4 seconds"), + Effect.orElseSucceed(() => null), + ); + const billing = yield* options.client.billing(binding).pipe( + Effect.timeout("4 seconds"), + Effect.orElseSucceed(() => null), + ); + const snapshot = { + repository: options.repository, + branch: options.branch, + sessionId: binding.cloudAgentSessionId, + worktreeId: binding.worktreeId, + task: taskState, + sandbox: sandbox?.status ?? ("unknown" as const), + billing: billing?.phase ?? ("unknown" as const), + billingAttribution: billing?.attribution ?? null, + estimatedHourlyRateUsd: + billing?.estimatedHourlyRateMicrodollars == null + ? null + : billing.estimatedHourlyRateMicrodollars / 1_000_000, + }; + const signature = encode(snapshot); + if (signature !== lifecycleSignature) { + lifecycleSignature = signature; + thread = { + ...thread, + nativeMetadata: { + ...thread.nativeMetadata, + cloudExecution: { ...snapshot, observedAt: DateTime.formatIso(yield* DateTime.now) }, + }, + }; + yield* emit({ type: "provider_thread.updated", driver, providerThread: thread }); + } + if (!active && sandbox?.status === "sleeping" && billing?.phase === "idle") + monitorSandbox = false; + }); + const watchEvents = Effect.gen(function* () { + if (streamWatching || !binding) return; + streamWatching = true; + const ownedBinding = binding; + yield* Effect.gen(function* () { + // State is changed by the reconciler while this reader observes notifications. + // oxlint-disable-next-line no-unmodified-loop-condition + while (active || needsHistoryRestore || monitorSandbox) { + yield* options.client.events(ownedBinding, streamCursor).pipe( + Stream.runForEach((event) => + Effect.gen(function* () { + streamCursor = Math.max(streamCursor, event.eventId); + if ( + event.streamEventType.startsWith("cloud.message.") || + (event.streamEventType === "kilocode" && + [ + "permission.asked", + "question.asked", + "session.error", + "session.idle", + ].includes(String(event.data.type))) + ) + yield* Queue.offer(wake, undefined); + }), + ), + Effect.catch(() => + status( + "Cloud stream disconnected. Polling continues; remote execution and billing were not stopped.", + ), + ), + ); + if (active || needsHistoryRestore || monitorSandbox) yield* Effect.sleep("5 seconds"); + } + streamWatching = false; + }).pipe(Effect.forkIn(scope)); + }); + const watch = Effect.gen(function* () { + if (watching) { + yield* Queue.offer(wake, undefined); + return; + } + watching = true; + yield* Effect.gen(function* () { + let polls = 0; + // Reconcile and lifecycle update this session state. + // oxlint-disable-next-line no-unmodified-loop-condition + while (active || needsHistoryRestore || monitorSandbox) { + yield* watchEvents; + if (active || needsHistoryRestore) + yield* gate + .withPermit(reconcile().pipe(Effect.timeout("10 seconds"))) + .pipe( + Effect.catch(() => + status( + "Cloud connection unavailable. Task and billing status are unknown; no prompt was resubmitted.", + ), + ), + ); + if (binding && (polls++ % 15 === 0 || !active)) + yield* gate.withPermit(lifecycle.pipe(Effect.timeout("10 seconds"), Effect.ignore)); + if (active || needsHistoryRestore || monitorSandbox) + yield* Effect.raceFirst( + Effect.sleep(active ? "2 seconds" : "15 seconds"), + Queue.take(wake), + ); + } + watching = false; + }).pipe(Effect.forkIn(scope)); + }); + const policyHash = (policy: Adapter.ProviderAdapterV2RuntimePolicy) => + wire( + crypto.digest( + "SHA-256", + new TextEncoder().encode( + encode({ + rules: openCodePermissionRules(policy), + interactionMode: policy.interactionMode, + }), + ), + ), + ).pipe(Effect.map(Encoding.encodeHex)); + const bind = Effect.fn("KiloCloudAdapterV2.bind")(function* ( + saved: OrchestrationV2ProviderThread, + ) { + if ( + saved.providerInstanceId !== options.instanceId || + (saved.nativeMetadata?.continuationKey && + saved.nativeMetadata.continuationKey !== options.continuationKey) + ) + return yield* error("Kilo Cloud account or repository changed; start a separate thread."); + thread = { + ...saved, + providerSessionId: input.providerSessionId, + nativeMetadata: { + ...saved.nativeMetadata, + continuationKey: options.continuationKey, + cloudExecution: saved.nativeMetadata?.cloudExecution ?? { + repository: options.repository, + branch: options.branch, + sessionId: null, + worktreeId: null, + task: "not_started", + sandbox: "unknown", + billing: "unknown", + billingAttribution: null, + estimatedHourlyRateUsd: null, + observedAt: null, + }, + }, + }; + const entries = yield* wire(options.journal.readThread(saved.id)); + const last = entries.at(-1); + if ( + last && + (last.accountId !== options.accountId || + last.repository !== options.repository || + last.branch !== options.branch) + ) + return yield* error("Cloud journal belongs to another account or repository."); + binding = last?.binding ?? undefined; + taskState = last?.state === "active" ? "unknown" : (last?.state ?? "not_started"); + monitorSandbox = !!last?.binding; + active = + last && (last.state === "active" || last.state === "admission_unknown") + ? last + : undefined; + if (binding) thread = { ...thread, nativeThreadRef: nativeRef(binding.kiloSessionId) }; + return thread; + }); + const unsupported = () => + Effect.fail(error("This capability is unavailable for Kilo Cloud sessions.")); + const runtime: Adapter.ProviderAdapterV2SessionRuntime = { + instanceId: options.instanceId, + driver, + providerSessionId: input.providerSessionId, + providerSession: session, + events: Stream.fromEffectRepeat(Queue.take(queue)), + hasPendingBackgroundWork: Effect.sync( + () => active !== undefined || needsHistoryRestore || monitorSandbox, + ), + hasPendingBackgroundWorkForThread: (candidate) => + Effect.sync( + () => + candidate.id === thread?.id && + (active !== undefined || needsHistoryRestore || monitorSandbox), + ), + ensureThread: (request) => + gate.withPermit( + Effect.gen(function* () { + const existing = request.existingProviderThread; + if (!existing) return yield* error("Kilo Cloud requires a preallocated T3 thread."); + return yield* bind(existing); + }), + ), + resumeThread: (request) => gate.withPermit(bind(request.providerThread)), + startTurn: (request) => + gate.withPermit( + Effect.gen(function* () { + yield* owned(request.providerThread); + if (request.reattach) { + const saved = (yield* wire( + options.journal.readThread(request.providerThread.id), + )).find( + (entry) => + entry.providerThread.id === request.providerThread.id && + entry.providerTurn.runAttemptId === request.attemptId, + ); + if (!saved) + return yield* error( + "No durable cloud intent exists for this run. No task was resubmitted.", + ); + active = saved; + binding = saved.binding ?? undefined; + monitorSandbox = true; + if ( + saved.state === "completed" || + saved.state === "failed" || + saved.state === "interrupted" + ) { + needsHistoryRestore = true; + yield* reconcile().pipe(Effect.timeout("10 seconds"), Effect.ignore); + if (active) yield* finish(saved.state); + } + yield* watch; + return; + } + if (active) + return yield* error( + "A cloud task is active or its admission is unknown. Do not resubmit.", + ); + if (options.allowAdmission === false) + return yield* error( + "Paid cloud execution is disabled. Existing tasks can still be recovered and interrupted.", + ); + const rules = openCodePermissionRules(request.runtimePolicy); + if ( + request.runtimePolicy.runtimeMode !== "full-access" || + rules.some((rule) => rule.action !== "allow") + ) + return yield* error( + "This Kilo Cloud runtime cannot enforce restricted permissions or disable subagents. Select Full access explicitly or use local Kilo. No paid task was submitted.", + ); + if (request.runtimePolicy.interactionMode !== "default") + return yield* error( + "Plan mode is not yet supported for Kilo Cloud. No task was submitted.", + ); + const selectedPolicyHash = yield* policyHash(request.runtimePolicy); + const priorIntent = (yield* wire(options.journal.read)).findLast( + (entry) => entry.providerThread.id === request.providerThread.id, + ); + if (priorIntent && priorIntent.policyHash !== selectedPolicyHash) + return yield* error( + "Cloud permissions changed. Start a separate cloud thread; the remote agent retains its original permissions.", + ); + terminalSignal = yield* Deferred.make(); + if (request.message.attachments.length) + return yield* error( + "Cloud attachments are not supported; no local files were uploaded.", + ); + const operationKey = yield* wire(crypto.randomUUIDv4); + const at = yield* DateTime.now; + const messageId = `msg_${DateTime.toEpochMillis(at).toString(16).padStart(12, "0")}${operationKey.replaceAll("-", "").slice(0, 14)}`; + const model = request.modelSelection.model; + const variant = getModelSelectionStringOptionValue(request.modelSelection, "variant"); + const payload = { + operationKey, + initialMessageId: messageId, + prompt: request.message.text, + repository: options.repository, + branch: options.branch, + model, + ...(variant ? { variant } : {}), + }; + const hash = yield* wire( + crypto.digest("SHA-256", new TextEncoder().encode(encode(payload))), + ); + const providerThread = { + ...request.providerThread, + nativeMetadata: { + ...request.providerThread.nativeMetadata, + continuationKey: options.continuationKey, + turnCorrelations: { + [messageId]: { + messageId: request.message.messageId, + nodeId: request.rootNodeId, + runId: request.runId, + attemptId: request.attemptId, + ordinal: request.providerTurnOrdinal, + attachments: [], + createdBy: request.message.createdBy, + creationSource: request.message.creationSource, + }, + }, + }, + }; + const intent: Journal.CloudIntent = { + revision: 0, + accountId: options.accountId, + repository: options.repository, + branch: options.branch, + operationKey, + messageId, + payloadHash: Encoding.encodeHex(hash), + policyHash: selectedPolicyHash, + binding: binding ?? null, + prepared: null, + state: "admission_unknown", + interruptRequested: false, + answeredRequestIds: [], + providerThread, + providerTurn: { + id: ids.derive.providerTurn({ driver, nativeTurnId: key(messageId) }), + providerThreadId: providerThread.id, + nodeId: request.rootNodeId, + runAttemptId: request.attemptId, + nativeTurnRef: nativeRef(messageId), + ordinal: request.providerTurnOrdinal, + status: "running", + startedAt: at, + completedAt: null, + }, + runOrdinal: request.runOrdinal, + }; + if (!(yield* wire(options.journal.reserve(intent)))) + return yield* error("A previous cloud admission still needs reconciliation."); + active = intent; + taskState = "admission_unknown"; + monitorSandbox = true; + thread = providerThread; + if (thread.nativeMetadata?.cloudExecution) { + thread = { + ...thread, + nativeMetadata: { + ...thread.nativeMetadata, + cloudExecution: { + ...thread.nativeMetadata.cloudExecution, + task: taskState, + sandbox: "unknown", + billing: "unknown", + billingAttribution: null, + estimatedHourlyRateUsd: null, + observedAt: null, + }, + }, + }; + lifecycleSignature = ""; + } + yield* emit({ type: "provider_thread.updated", driver, providerThread: thread }); + yield* emit({ + type: "provider_turn.updated", + driver, + providerTurn: intent.providerTurn, + }); + let submissionConfirmed = false; + yield* Effect.gen(function* () { + if (binding) + yield* options.client.send(binding, { + messageId, + prompt: request.message.text, + model, + ...(variant ? { variant } : {}), + }); + else { + const prepared = yield* options.client.prepare(payload); + submissionConfirmed = true; + yield* save({ ...intent, prepared }); + binding = yield* options.client.bind( + prepared, + options.repository, + messageId, + options.branch, + ); + } + yield* save({ ...active!, binding, state: "active" }); + thread = { ...thread!, nativeThreadRef: nativeRef(binding.kiloSessionId) }; + yield* emit({ type: "provider_thread.updated", driver, providerThread: thread }); + }).pipe( + Effect.catch((cause) => + !submissionConfirmed && isCloudError(cause) && cause.reason === "rejected" + ? finish("failed") + : status( + "Cloud admission is uncertain. Its operation ID is saved; no automatic retry will start another paid task.", + ), + ), + ); + yield* watch; + }), + ), + interruptTurn: (request) => + gate + .withPermit( + Effect.gen(function* () { + yield* owned(request.providerThread); + if (!active || active.providerTurn.id !== request.providerTurnId) return false; + if (!binding) + return yield* error( + "Cloud admission has no confirmed session ID. Remote Stop is unavailable; task and billing status remain unknown.", + ); + yield* watch; + if (active.interruptRequested) return true; + yield* save({ ...active, interruptRequested: true }); + const accepted = yield* wire( + options.client + .interrupt(binding) + .pipe( + Effect.catch((cause) => + cause.reason === "admission_unknown" + ? Effect.fail(cause) + : save({ ...active!, interruptRequested: false }).pipe( + Effect.andThen(Effect.fail(cause)), + ), + ), + ), + ); + if (!accepted.success) yield* save({ ...active!, interruptRequested: false }); + yield* status( + accepted.success + ? "Interrupt requested. Waiting for remote task confirmation; sandbox and billing may remain active." + : "Kilo did not confirm interruption. The remote task may still be running.", + ); + return true; + }), + ) + .pipe( + Effect.flatMap((wait) => + wait + ? Deferred.await(terminalSignal).pipe( + Effect.timeout("45 seconds"), + Effect.mapError(() => + error( + "Remote interruption is still unconfirmed. The task may still be running; sandbox and billing status are separate.", + ), + ), + ) + : Effect.void, + ), + ), + readThreadSnapshot: (request) => + gate.withPermit( + Effect.gen(function* () { + yield* owned(request.providerThread); + yield* reconcile().pipe( + Effect.timeout("10 seconds"), + Effect.mapError(() => + error( + "Cloud history is temporarily unavailable. Remote execution may still be active.", + ), + ), + ); + const intents = yield* wire(options.journal.readThread(thread!.id)); + return { + providerThread: thread!, + providerTurns: intents.map((entry) => entry.providerTurn), + messages: [...messages.values()], + runtimeRequests: [...requests.values()].map((entry) => entry.runtime), + }; + }), + ), + respondToRuntimeRequest: (response) => + gate.withPermit( + Effect.gen(function* () { + const pending = requests.get(response.requestId); + if (!pending || pending.runtime.status !== "pending" || !active || !binding) + return yield* error("Cloud interaction is not pending for this turn."); + if (active.answeredRequestIds.includes(pending.native.id)) + return yield* error( + "The previous answer has an uncertain outcome; it was not resubmitted.", + ); + const current = yield* wire(options.client.pending(binding)); + if ( + ![...current.questions, ...current.permissions].some( + (request) => request.id === pending.native.id, + ) + ) + return yield* error("The cloud interaction has already ended."); + let reply; + if ("questions" in pending.native) { + const answers = pending.native.questions.map((_, index) => { + const value = response.answers?.[String(index)]; + return typeof value === "string" + ? [value] + : Array.isArray(value) && value.every((answer) => typeof answer === "string") + ? value + : []; + }); + if (answers.some((answer) => answer.length === 0)) + return yield* error("Each cloud question requires an answer."); + reply = options.client.replyQuestion(binding, pending.native.id, answers); + } else { + if (!response.decision) + return yield* error("A cloud permission decision is required."); + reply = options.client.replyPermission( + binding, + pending.native.id, + response.decision === "accept" + ? "once" + : response.decision === "acceptForSession" || + response.decision === "acceptAlways" + ? "always" + : "reject", + ); + } + yield* save({ + ...active, + answeredRequestIds: [...active.answeredRequestIds, pending.native.id], + }); + const resetAnswer = () => + save({ + ...active!, + answeredRequestIds: active!.answeredRequestIds.filter( + (id) => id !== pending.native.id, + ), + }); + const accepted = yield* wire( + reply.pipe( + Effect.catch((cause) => + cause.reason === "admission_unknown" + ? Effect.fail(cause) + : resetAnswer().pipe(Effect.andThen(Effect.fail(cause))), + ), + ), + ); + if (!accepted.success) { + yield* resetAnswer(); + return yield* error( + "Kilo did not confirm delivery of the answer. Task status remains unknown.", + ); + } + yield* resolveRequest(pending); + }), + ), + steerTurn: unsupported, + rollbackThread: unsupported, + forkThread: unsupported, + }; + return runtime; + }), + }); +}); diff --git a/apps/server/src/orchestration-v2/EffectOutbox.ts b/apps/server/src/orchestration-v2/EffectOutbox.ts index 4841a5762480..4394f8fe9462 100644 --- a/apps/server/src/orchestration-v2/EffectOutbox.ts +++ b/apps/server/src/orchestration-v2/EffectOutbox.ts @@ -24,6 +24,7 @@ import * as Schema from "effect/Schema"; import * as SqlClient from "effect/unstable/sql/SqlClient"; export const OrchestrationEffectRequestV2 = Schema.Union([ + Schema.Struct({ type: Schema.Literal("provider-turn.reattach"), runId: RunId }), Schema.Struct({ type: Schema.Literal("provider-runtime.continue"), sourceRunId: RunId, @@ -106,6 +107,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([ export type OrchestrationEffectRequestV2 = typeof OrchestrationEffectRequestV2.Type; export const REPLAY_SAFE_EFFECT_TYPES_AFTER_PROCESS_LOSS = [ + "provider-turn.reattach", "provider-runtime.continue", "provider-session.detach", "provider-thread.rollback", diff --git a/apps/server/src/orchestration-v2/EffectWorker.ts b/apps/server/src/orchestration-v2/EffectWorker.ts index 1abf070b87ec..f6f8e8a54e46 100644 --- a/apps/server/src/orchestration-v2/EffectWorker.ts +++ b/apps/server/src/orchestration-v2/EffectWorker.ts @@ -144,9 +144,15 @@ export const executorLayer: Layer.Layer< }), ), ); + case "provider-turn.reattach": case "provider-turn.start": return providerTurnStart - .start({ threadId: effect.threadId, runId: effect.request.runId, willRetry }) + .start({ + threadId: effect.threadId, + runId: effect.request.runId, + willRetry, + ...(effect.request.type === "provider-turn.reattach" ? { reattach: true } : {}), + }) .pipe( Effect.mapError( (cause) => diff --git a/apps/server/src/orchestration-v2/ProviderAdapter.ts b/apps/server/src/orchestration-v2/ProviderAdapter.ts index ac300004322e..5c7d1261ca28 100644 --- a/apps/server/src/orchestration-v2/ProviderAdapter.ts +++ b/apps/server/src/orchestration-v2/ProviderAdapter.ts @@ -394,6 +394,8 @@ export interface ProviderAdapterV2EnsureThreadInput { } export interface ProviderAdapterV2TurnInput { + /** Reattach a durable remote turn without sending the prompt again. */ + readonly reattach?: boolean; readonly appThread: OrchestrationV2AppThread; readonly threadId: ThreadId; readonly runId: RunId; diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts index 8df85818a761..234651d054c1 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts @@ -1272,3 +1272,134 @@ it.effect( }).pipe(Effect.provide(layer)); }, ); + +it.effect( + "preserves remote work and deduplicates reattach while still cancelling local work", + () => { + const threadId = ThreadId.make("mixed-cloud-local"); + const cloudThread = ProviderThreadId.make("remote-thread"); + const localThread = ProviderThreadId.make("local-thread"); + const cloudTurn = ProviderTurnId.make("remote-turn"); + const localTurn = ProviderTurnId.make("local-turn"); + const cloudRun = RunId.make("remote-run"); + const localRun = RunId.make("local-run"); + const cloudInstance = ProviderInstanceId.make("cloud-instance"); + const localInstance = ProviderInstanceId.make("local-instance"); + const projection = { + thread: { id: threadId, providerInstanceId: cloudInstance }, + providerThreads: [ + { + id: cloudThread, + driver: ProviderDriverKind.make("kilo-cloud"), + providerInstanceId: cloudInstance, + }, + { + id: localThread, + driver: ProviderDriverKind.make("kilo"), + providerInstanceId: localInstance, + }, + ], + providerSessions: [], + runs: [ + { + id: cloudRun, + providerThreadId: cloudThread, + providerInstanceId: cloudInstance, + status: "running", + rootNodeId: null, + activeAttemptId: RunAttemptId.make("cloud-attempt"), + }, + { + id: localRun, + providerThreadId: localThread, + providerInstanceId: localInstance, + status: "running", + rootNodeId: null, + }, + ], + providerTurns: [ + { id: cloudTurn, providerThreadId: cloudThread, status: "running" }, + { id: localTurn, providerThreadId: localThread, status: "running" }, + ], + runtimeRequests: [ + { + id: RuntimeRequestId.make("remote-approval"), + providerTurnId: cloudTurn, + nodeId: NodeId.make("remote-node"), + status: "pending", + responseCapability: { type: "live" }, + }, + ], + attempts: [], + nodes: [], + subagents: [], + messages: [], + turnItems: [], + } as unknown as OrchestrationV2ThreadProjection; + const events: OrchestrationV2ThreadProjection["runs"][number][] = []; + const requests: string[] = []; + const pending: EffectOutbox.PendingOrchestrationEffectV2[] = []; + const record = (input: Parameters[0]) => { + for (const event of input.events) { + if (event.type === "run.updated") events.push(event.payload); + if (event.type === "runtime-request.updated") requests.push(event.payload.id); + } + pending.push(...input.effects); + }; + const layer = ProviderRuntimeRecovery.layer.pipe( + Layer.provide(ServerSettings.layerTest()), + Layer.provide( + Layer.mergeAll( + IdAllocator.layer, + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getRecoveryThreadIds: () => Effect.succeed([threadId]), + getRuntimeRecoveryProjection: () => Effect.succeed(projection), + }), + Layer.mock(EventSink.EventSinkV2)({ + writeWithEffects: (input) => + Effect.sync(() => { + record(input); + return [] as never; + }), + commitCommand: (input) => + Effect.sync(() => { + record({ events: input.events, effects: input.effects }); + return { committed: true, cancelledEffectCount: 0 } as never; + }), + }), + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({ + runRecoveryOnce: Effect.succeed(false), + }), + Layer.mock(EffectOutbox.EffectOutboxV2)({ + listByCommandId: (id) => + Effect.sync( + () => + pending + .filter((effect) => effect.commandId === id) + .map((effect) => ({ ...effect, status: "pending" })) as never, + ), + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), + cancelUnsettled: () => Effect.succeed([]), + signalCancellations: () => Effect.void, + }), + ), + ), + ); + return Effect.gen(function* () { + const service = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService; + yield* service.reconcile("shutdown"); + assert.isTrue(events.some((run) => run.id === localRun && run.status === "cancelled")); + assert.isFalse(events.some((run) => run.id === cloudRun)); + assert.equal(requests.length, 0); + yield* service.reconcile("startup"); + yield* service.reconcile("startup"); + assert.equal( + pending.filter((effect) => effect.request.type === "provider-turn.reattach").length, + 1, + ); + assert.isTrue(events.some((run) => run.id === cloudRun && run.status === "starting")); + assert.isFalse(events.some((run) => run.id === cloudRun && run.status === "cancelled")); + assert.equal(requests.length, 0); + }).pipe(Effect.provide(layer)); + }, +); diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 67ae22f1883b..44adf2516902 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -186,6 +186,117 @@ export const make = Effect.gen(function* () { continueAfterRestart: boolean, ) { const now = yield* DateTime.now; + // Remote execution survives this process. Keep its turns and requests; + // startup reattaches journal-bound observers without sending a new prompt. + const cloudThreads = new Set( + projection.providerThreads + .filter((thread) => thread.driver === "kilo-cloud") + .map((thread) => thread.id), + ); + const cloudRuns = nonterminalRuns(projection).filter( + (run) => run.providerThreadId !== null && cloudThreads.has(run.providerThreadId), + ); + if (cloudThreads.size > 0) { + const events: OrchestrationV2DomainEvent[] = []; + const effects: EffectOutbox.PendingOrchestrationEffectV2[] = []; + for (const run of cloudRuns) { + const commandId = CommandId.make( + `command:cloud-reattach:${run.id}:${run.activeAttemptId}`, + ); + if (run.status !== "queued" && trigger !== "startup") continue; + if (run.status === "queued" && run.queueHeld === true) continue; + if (run.status !== "queued") { + const existing = yield* outbox + .listByCommandId(commandId) + .pipe( + Effect.mapError( + (cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }), + ), + ); + if ( + existing.some( + (effect) => + effect.request.type === "provider-turn.reattach" && + (effect.status === "pending" || effect.status === "running"), + ) + ) + continue; + effects.push({ + id: `effect:cloud-reattach:${run.id}:${run.activeAttemptId}:${DateTime.formatIso(now)}`, + commandId, + threadId: projection.thread.id, + request: { type: "provider-turn.reattach", runId: run.id }, + }); + } + events.push({ + id: yield* ids.allocate + .event({ threadId: projection.thread.id, commandId }) + .pipe( + Effect.mapError( + (cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }), + ), + ), + type: "run.updated", + threadId: projection.thread.id, + runId: run.id, + ...(run.rootNodeId ? { nodeId: run.rootNodeId } : {}), + providerInstanceId: run.providerInstanceId, + occurredAt: now, + payload: + run.status === "queued" + ? { ...run, queueHeld: true } + : { ...run, status: "starting" }, + }); + } + if (events.length || effects.length) + yield* eventSink.writeWithEffects({ events, effects }).pipe( + Effect.mapError( + (cause) => + new ProviderRuntimeRecoveryError({ + operation: "reconcile", + threadId: projection.thread.id, + cause, + }), + ), + ); + const remoteRunIds = new Set( + projection.runs + .filter( + (run) => run.providerThreadId !== null && cloudThreads.has(run.providerThreadId), + ) + .map((run) => run.id), + ); + const remoteTurnIds = new Set( + projection.providerTurns + .filter((turn) => cloudThreads.has(turn.providerThreadId)) + .map((turn) => turn.id), + ); + projection = { + ...projection, + runs: projection.runs.filter((run) => !remoteRunIds.has(run.id)), + attempts: projection.attempts.filter((attempt) => !remoteRunIds.has(attempt.runId)), + nodes: projection.nodes.filter( + (node) => node.runId === null || !remoteRunIds.has(node.runId), + ), + subagents: projection.subagents.filter( + (agent) => agent.runId === null || !remoteRunIds.has(agent.runId), + ), + providerSessions: projection.providerSessions.filter( + (session) => session.driver !== "kilo-cloud", + ), + providerThreads: projection.providerThreads.filter( + (thread) => !cloudThreads.has(thread.id), + ), + providerTurns: projection.providerTurns.filter((turn) => !remoteTurnIds.has(turn.id)), + runtimeRequests: projection.runtimeRequests.filter( + (request) => + request.providerTurnId === null || !remoteTurnIds.has(request.providerTurnId), + ), + turnItems: projection.turnItems.filter( + (item) => item.runId === null || !remoteRunIds.has(item.runId), + ), + }; + } const runs = [] as Array; for (const run of nonterminalRuns(projection)) { if (run.status === "waiting") { diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 786eb5a6bf0a..b843fd7c4ec9 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -1550,8 +1550,18 @@ export const layerWithOptions = ( sessionOpen.withLock( input.providerSessionId, Effect.gen(function* () { + const adapter = yield* registry.get(input.modelSelection.instanceId).pipe( + Effect.mapError( + (cause) => + new ProviderSessionOpenError({ + instanceId: input.modelSelection.instanceId, + providerSessionId: input.providerSessionId, + cause, + }), + ), + ); const cwd = input.runtimePolicy.cwd; - if (cwd !== null) { + if (cwd !== null && adapter.driver !== "kilo-cloud") { const workspaceIsDirectory = yield* fileSystem.stat(cwd).pipe( Effect.map((stat) => stat.type === "Directory"), Effect.catch((error) => Effect.succeed(error.reason._tag !== "NotFound")), @@ -1585,20 +1595,10 @@ export const layerWithOptions = ( return existing.exposedRuntime; } - const adapter = yield* registry.get(input.modelSelection.instanceId).pipe( - Effect.mapError( - (cause) => - new ProviderSessionOpenError({ - instanceId: input.modelSelection.instanceId, - providerSessionId: input.providerSessionId, - cause, - }), - ), - ); - const prepared = yield* prepareMcpSession( - input.threadId, - input.modelSelection.instanceId, - ); + const prepared: PreparedMcpCredential = + adapter.driver === "kilo-cloud" + ? { mcpCredentialId: undefined, issued: false } + : yield* prepareMcpSession(input.threadId, input.modelSelection.instanceId); const mcpCredentialId = prepared.mcpCredentialId; // The reservation from prepare protects the credential (which // eager adapters bake into the provider process during diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts index 47f7aa80bdc4..b97234e64a9e 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.ts @@ -136,6 +136,14 @@ export const layer: Layer.Layer< } const session = yield* sessions.get(input.providerSessionId); if (Option.isNone(session)) { + if (providerThread.driver === "kilo-cloud") + return yield* new ProviderTurnControlError({ + threadId: input.threadId, + operation: input.operation, + providerTurnId: input.providerTurnId, + cause: + "Kilo Cloud is disconnected. Reconnect to confirm remote interruption; the task and billing may still be active.", + }); // Interrupt/restart against a already-released session must not fail // the durable effect (and retry 5x). The turn may still look running // in projection until recovery/finalization; there is no live adapter diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 781491ac7fd6..523d87798b14 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -73,6 +73,7 @@ export interface ProviderTurnStartServiceV2Shape { readonly threadId: ThreadId; readonly runId: RunId; readonly willRetry?: boolean; + readonly reattach?: boolean; }) => Effect.Effect; } @@ -217,6 +218,7 @@ export const layer: Layer.Layer< readonly threadId: ThreadId; readonly runId: RunId; readonly willRetry?: boolean; + readonly reattach?: boolean; }) { const { runId } = input; const projection = yield* projectionStore.getTurnStartContext(input.threadId, runId); @@ -454,8 +456,33 @@ export const layer: Layer.Layer< return; } } + if ( + providerThread.driver === "kilo-cloud" && + (message.attachments.length > 0 || + (message.context?.records.length ?? 0) > 0 || + handoffs.length > 0 || + nativeForkTransfer !== undefined) + ) { + yield* settleRunBeforeStart({ + signal: "cloud-context-rejected", + status: "failed", + now: yield* DateTime.now, + providerInstanceId: run.providerInstanceId, + itemProviderThreadId: providerThread.id, + item: { + type: "error", + title: "Local context cannot be sent to Kilo Cloud", + failure: makeProviderFailure({ + class: "validation_error", + message: + "Kilo Cloud does not accept local files, composer context, forks or handoff history. Send a plain prompt in a separate cloud thread.", + }), + }, + }); + return; + } const { worktreePath, branch } = projection.thread; - if (worktreePath !== null && branch !== null) { + if (providerThread.driver !== "kilo-cloud" && worktreePath !== null && branch !== null) { const exists = yield* fileSystem .exists(worktreePath) .pipe(Effect.orElseSucceed(() => true)); @@ -576,7 +603,8 @@ export const layer: Layer.Layer< }); }); if (sessionResult._tag === "Failure") { - if (input.willRetry === true) return yield* sessionResult.failure; + if (input.willRetry === true || input.reattach === true) + return yield* sessionResult.failure; yield* settleStartFailure({ signal: "provider-session-open-failure", title: "Provider session failed to open", @@ -593,7 +621,7 @@ export const layer: Layer.Layer< Effect.gen(function* () { const loaded = yield* Effect.result(load); if (loaded._tag === "Success") return loaded.success; - if (input.willRetry === true) return yield* loaded.failure; + if (input.willRetry === true || input.reattach === true) return yield* loaded.failure; yield* settleStartFailure({ signal: "provider-thread-load-failure", title: "Provider turn failed to start", @@ -1208,6 +1236,7 @@ export const layer: Layer.Layer< ? session : makeDeliverySession(session, startWithHandoffs); yield* runExecution.startRootRun({ + ...(input.reattach ? { reattach: true } : {}), commandId: CommandId.make(`command:effect:provider-turn.start:${run.id}`), appThread: projection.thread, providerSessionId, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 370c30058346..a0bda9c42434 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -544,6 +544,7 @@ it.effect("rechecks run ownership immediately before calling the provider", () = providerTurnId: null, } as OrchestrationV2RunAttempt; const session = { + providerSession: { capabilities: { checkpointing: { appCanCheckpointFilesystem: true } } }, events: Stream.never, startTurn: () => Ref.update(providerStarts, (count) => count + 1), } as unknown as ProviderAdapterV2SessionRuntime; @@ -616,6 +617,9 @@ it.effect( appThread: { id: threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:compact-routing:${index}`), session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.never, startTurn: () => Effect.sync(() => { @@ -690,6 +694,9 @@ it.effect("refreshes MCP credential liveness before calling the provider", () => appThread: { id: threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make("session:run-execution-mcp-liveness"), session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.never, startTurn: () => Ref.update(order, (entries) => [...entries, "start-turn"]), } as unknown as ProviderAdapterV2SessionRuntime, @@ -801,6 +808,9 @@ it.effect("starts the provider when checkpoint baseline capture fails", () => appThread: { id: threadId } as OrchestrationV2AppThread, providerSessionId, session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.never, startTurn: () => Ref.update(providerStarts, (count) => count + 1), } as unknown as ProviderAdapterV2SessionRuntime, @@ -937,6 +947,9 @@ it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as co appThread: { id: threadId } as OrchestrationV2AppThread, providerSessionId, session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.never, startTurn: () => Ref.update(providerStarts, (count) => count + 1), } as unknown as ProviderAdapterV2SessionRuntime, @@ -1186,6 +1199,9 @@ it.effect("keeps ingesting owned child events after the root turn terminalizes", appThread: { id: threadId } as OrchestrationV2AppThread, providerSessionId, session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.fromIterable(events), startTurn: () => Effect.void, } as unknown as ProviderAdapterV2SessionRuntime, @@ -1566,6 +1582,9 @@ it.effect( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${key}`), session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.empty, // Session-wide stays true forever; the root must consult the // thread-scoped probe instead of being pinned by siblings. @@ -1783,6 +1802,9 @@ it.effect("drops late root provider-thread writes from a superseded attempt", () appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${key}`), session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.empty, hasPendingBackgroundWork: Effect.succeed(true), hasPendingBackgroundWorkForThread: () => Effect.succeed(true), @@ -1975,6 +1997,9 @@ it.effect( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${key}`), session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.empty, hasPendingBackgroundWork: Effect.succeed(true), hasPendingBackgroundWorkForThread: () => Effect.succeed(true), @@ -2135,6 +2160,9 @@ it.effect( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${key}`), session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.empty, // Session-wide stays true (sibling has work). Stop must use only // the scoped probe for this root's provider thread. @@ -2319,6 +2347,9 @@ it.effect( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make("session:subagent-interrupt-cascade"), session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.empty, subscribeEvents: Effect.succeed({ events: Stream.fromIterable([ @@ -2673,6 +2704,9 @@ it.effect( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make("session:subagent-link-survives-terminal"), session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.empty, subscribeEvents: Effect.succeed({ events: Stream.fromIterable([ @@ -3332,6 +3366,9 @@ function captureRootRunTermination(input: { appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${input.key}`), session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.empty, subscribeEvents: Effect.succeed({ events: @@ -3800,6 +3837,9 @@ function runBackgroundItemScenario( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${key}`), session: { + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, events: Stream.empty, subscribeEvents: Effect.gen(function* () { yield* options?.onSubscribe ?? Effect.void; diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 71211c126545..3b22014436c6 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -505,6 +505,7 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly attempt: OrchestrationV2RunAttempt; readonly attemptId: RunAttemptId; readonly providerTurnOrdinal: number; + readonly reattach?: boolean; readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect< ReadonlyArray, unknown @@ -562,6 +563,7 @@ export const layer: Layer.Layer< readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection; readonly terminal: ProviderTerminalEvent; readonly failureItemPersisted: boolean; + readonly checkpointFilesystem: boolean; readonly refreshAfterTurn: Effect.Effect; readonly writeIfRunCurrent?: { readonly activeAttemptId: RunAttemptId; @@ -631,7 +633,9 @@ export const layer: Layer.Layer< }) : []; const persistedStatus = - input.terminal.status === "completed" ? "waiting" : input.terminal.status; + input.terminal.status === "completed" && input.checkpointFilesystem + ? "waiting" + : input.terminal.status; // Completion cohorts are advanced by Orchestrator while a provider // turn is in flight. Do not replay the run snapshot captured at start // over a newer acknowledgement, successor, or Stop barrier. @@ -640,13 +644,19 @@ export const layer: Layer.Layer< const finalizedRun: OrchestrationV2Run = { ...runWithoutDelegatedCompletion, status: persistedStatus, - completedAt: input.terminal.status === "completed" ? null : completedAt, + completedAt: + input.terminal.status === "completed" && input.checkpointFilesystem + ? null + : completedAt, }; const finalizedRootNode: OrchestrationV2ExecutionNode = { ...input.rootNode, status: persistedStatus, - completedAt: input.terminal.status === "completed" ? null : completedAt, - checkpointScopeId: input.checkpointScope.id, + completedAt: + input.terminal.status === "completed" && input.checkpointFilesystem + ? null + : completedAt, + checkpointScopeId: input.checkpointFilesystem ? input.checkpointScope.id : null, }; const finalizedProviderThread: OrchestrationV2ProviderThread = { ...input.providerThread, @@ -664,9 +674,10 @@ export const layer: Layer.Layer< // ahead of any later run's start on this thread's effect lane. const finalization = { effects: - input.terminal.status === "completed" || - input.terminal.status === "interrupted" || - input.terminal.status === "cancelled" + input.checkpointFilesystem && + (input.terminal.status === "completed" || + input.terminal.status === "interrupted" || + input.terminal.status === "cancelled") ? [ { id: `effect:checkpoint.capture:${input.run.id}`, @@ -793,9 +804,14 @@ export const layer: Layer.Layer< return RunExecutionServiceV2.of({ startRootRun: (input) => Effect.gen(function* () { + const checkpointFilesystem = + input.session.providerSession.capabilities.checkpointing.appCanCheckpointFilesystem; // Startup failure and stream shutdown can report the same attempt. const refreshAfterTurn = yield* Effect.cached( - finalizationObserver.refreshAfterTurn(input.appThread.projectId).pipe( + (checkpointFilesystem + ? finalizationObserver.refreshAfterTurn(input.appThread.projectId) + : Effect.void + ).pipe( Effect.catchCause((cause) => Effect.logWarning("failed to refresh pull requests after run termination", { threadId: input.run.threadId, @@ -832,21 +848,22 @@ export const layer: Layer.Layer< .responseStreamingMode, ), ); - yield* checkpointService - .captureBaseline({ - scope: input.checkpointScope, - ordinalWithinScope: Math.max(0, input.run.ordinal - 1), - }) - .pipe( - Effect.catchCause((cause) => - Cause.hasInterruptsOnly(cause) - ? Effect.failCause(cause) - : Effect.logWarning( - "orchestration V2 checkpoint baseline capture failed; starting provider without a baseline", - { runId: input.run.id }, - ), - ), - ); + if (checkpointFilesystem) + yield* checkpointService + .captureBaseline({ + scope: input.checkpointScope, + ordinalWithinScope: Math.max(0, input.run.ordinal - 1), + }) + .pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.failCause(cause) + : Effect.logWarning( + "orchestration V2 checkpoint baseline capture failed; starting provider without a baseline", + { runId: input.run.id }, + ), + ), + ); if ( input.shouldStartProviderTurn !== undefined && !(yield* input.shouldStartProviderTurn()) @@ -865,6 +882,7 @@ export const layer: Layer.Layer< cause, }); yield* writeFinalRunEvents({ + checkpointFilesystem, run: input.run, rootNode: input.rootNode, checkpointScope: input.checkpointScope, @@ -961,6 +979,7 @@ export const layer: Layer.Layer< const providerThread = yield* Ref.get(latestProviderThread); const openSubagents = yield* Ref.get(openRunOwnedSubagents); yield* writeFinalRunEvents({ + checkpointFilesystem, run: input.run, rootNode: input.rootNode, checkpointScope: input.checkpointScope, @@ -1116,7 +1135,11 @@ export const layer: Layer.Layer< } const terminal = yield* Ref.get(terminalEvent); // Non-completed terminals drop background tracking immediately. - if (terminal !== null && terminal.status !== "completed") { + if ( + terminal !== null && + terminal.status !== "completed" && + input.session.driver !== "kilo-cloud" + ) { return true; } const childProviderTurns = yield* Ref.get(activeChildProviderTurns); @@ -1272,7 +1295,7 @@ export const layer: Layer.Layer< cause, }).pipe( Effect.andThen( - finalized + finalized || input.session.driver === "kilo-cloud" ? Effect.void : Ref.get(latestProviderThread).pipe( Effect.flatMap((providerThread) => @@ -1281,6 +1304,7 @@ export const layer: Layer.Layer< Ref.get(openRunOwnedSubagents).pipe( Effect.flatMap((openSubagents) => writeFinalRunEvents({ + checkpointFilesystem, run: input.run, rootNode: input.rootNode, checkpointScope: input.checkpointScope, @@ -1346,6 +1370,7 @@ export const layer: Layer.Layer< runId: input.run.id, runOrdinal: input.run.ordinal, providerTurnOrdinal: input.providerTurnOrdinal, + ...(input.reattach ? { reattach: true } : {}), ...(input.run.restartContinuationOfRunId === undefined ? {} : { @@ -1375,57 +1400,67 @@ export const layer: Layer.Layer< : input.session.startTurn(turnInput); yield* startTurn.pipe( Effect.catchCause((cause) => - Effect.logError("orchestration V2 provider turn start failed", { - runId: input.run.id, - cause, - }).pipe( - Effect.andThen(Fiber.interrupt(providerEventFiber)), - Effect.andThen(Ref.get(latestProviderThread)), - Effect.flatMap((providerThread) => - Ref.get(latestTurnItemOrdinal).pipe( - Effect.flatMap((latestItemOrdinal) => - Ref.get(openRunOwnedSubagents).pipe( - Effect.flatMap((openSubagents) => - writeFinalRunEvents({ - run: input.run, - rootNode: input.rootNode, - checkpointScope: input.checkpointScope, - providerThread, - attempt: input.attempt, - ...(input.shouldFinalizeRun === undefined - ? {} - : { shouldFinalizeRun: input.shouldFinalizeRun }), - ...(input.hasUnpairedRunInterruptRequest === undefined - ? {} - : { - hasUnpairedRunInterruptRequest: - input.hasUnpairedRunInterruptRequest, - }), - openRunOwnedSubagents: openSubagents, - terminal: makeFailedTerminalEvent( - makeProviderFailure({ - cause: Cause.squash(cause), - class: "provider_error", + input.session.driver === "kilo-cloud" && + (input.reattach || Cause.hasInterruptsOnly(cause)) + ? Effect.fail( + new RunExecutionStartError({ + commandId: input.commandId, + runId: input.run.id, + cause, + }), + ) + : Effect.logError("orchestration V2 provider turn start failed", { + runId: input.run.id, + cause, + }).pipe( + Effect.andThen(Fiber.interrupt(providerEventFiber)), + Effect.andThen(Ref.get(latestProviderThread)), + Effect.flatMap((providerThread) => + Ref.get(latestTurnItemOrdinal).pipe( + Effect.flatMap((latestItemOrdinal) => + Ref.get(openRunOwnedSubagents).pipe( + Effect.flatMap((openSubagents) => + writeFinalRunEvents({ + checkpointFilesystem, + run: input.run, + rootNode: input.rootNode, + checkpointScope: input.checkpointScope, + providerThread, + attempt: input.attempt, + ...(input.shouldFinalizeRun === undefined + ? {} + : { shouldFinalizeRun: input.shouldFinalizeRun }), + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: + input.hasUnpairedRunInterruptRequest, + }), + openRunOwnedSubagents: openSubagents, + terminal: makeFailedTerminalEvent( + makeProviderFailure({ + cause: Cause.squash(cause), + class: "provider_error", + }), + latestItemOrdinal + 1, + ), + failureItemPersisted: false, + refreshAfterTurn, }), - latestItemOrdinal + 1, ), - failureItemPersisted: false, - refreshAfterTurn, - }), + ), ), ), ), + Effect.mapError( + (writeCause) => + new RunExecutionStartError({ + commandId: input.commandId, + runId: input.run.id, + cause: { start: cause, write: writeCause }, + }), + ), ), - ), - Effect.mapError( - (writeCause) => - new RunExecutionStartError({ - commandId: input.commandId, - runId: input.run.id, - cause: { start: cause, write: writeCause }, - }), - ), - ), ), ); }), diff --git a/apps/server/src/provider/Drivers/KiloCloudDriver.ts b/apps/server/src/provider/Drivers/KiloCloudDriver.ts new file mode 100644 index 000000000000..0e9c24d33ece --- /dev/null +++ b/apps/server/src/provider/Drivers/KiloCloudDriver.ts @@ -0,0 +1,180 @@ +import { KiloCloudSettings, TextGenerationError, type ServerProvider } from "@t3tools/contracts"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Encoding from "effect/Encoding"; +import * as Path from "effect/Path"; +import * as PubSub from "effect/PubSub"; +import * as Schema from "effect/Schema"; +import type * as Scope from "effect/Scope"; +import * as Stream from "effect/Stream"; +import * as ServerConfig from "../../config.ts"; +import * as CloudAdapter from "../../orchestration-v2/Adapters/KiloCloudAdapterV2.ts"; +import * as Account from "../kilo/KiloCloudAccount.ts"; +import * as Cloud from "../kilo/KiloCloudWebClient.ts"; +import * as Journal from "../kilo/KiloCloudJournal.ts"; +import { ProviderDriverError } from "../Errors.ts"; +import type { ProviderDriver } from "../ProviderDriver.ts"; +import { buildServerProvider } from "../providerSnapshot.ts"; +import { makeManualOnlyProviderMaintenanceCapabilities } from "../providerMaintenance.ts"; + +const kind = CloudAdapter.KILO_CLOUD_PROVIDER; +const decode = Schema.decodeSync(KiloCloudSettings); +type Requirements = + | Effect.Services> + | Effect.Services> + | Effect.Services> + | ServerConfig.ServerConfig; +export type KiloCloudDriverEnv = Exclude; +export const KiloCloudDriver: ProviderDriver = { + driverKind: kind, + metadata: { displayName: "Kilo Cloud", supportsMultipleInstances: true }, + configSchema: KiloCloudSettings, + defaultConfig: () => decode({}), + create: (input) => + Effect.gen(function* () { + const server = yield* ServerConfig.ServerConfig; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const account = yield* Account.make(input.config.profileDirectory); + const credentials = yield* account.load; + const identity = [ + input.instanceId, + path.resolve(input.config.profileDirectory), + credentials.accountId, + input.config.repository, + input.config.branch, + ].join("\0"); + const digest = yield* crypto.digest("SHA-256", new TextEncoder().encode(identity)); + const continuationKey = `kilo-cloud:${Encoding.encodeHex(digest)}`; + const journal = yield* Journal.make( + path.join(server.stateDir, "providers", "kilo-cloud", Encoding.encodeHex(digest)), + ); + const client = Cloud.make({ ...credentials, credentials: account.load }); + const adapter = yield* CloudAdapter.make({ + instanceId: input.instanceId, + continuationKey, + accountId: credentials.accountId, + repository: input.config.repository, + branch: input.config.branch, + client, + journal, + allowAdmission: input.enabled && input.config.cloudConsent, + }); + const configured = + /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(input.config.repository) && + input.config.cloudConsent; + const changes = yield* PubSub.unbounded(); + const checkedAt = DateTime.formatIso(yield* DateTime.now); + let snapshot: ServerProvider = { + ...buildServerProvider({ + driver: kind, + enabled: input.enabled, + presentation: { + displayName: "Kilo Cloud", + badgeLabel: "Preview", + supportsConversationRollback: false, + supportedRuntimeModes: ["full-access"], + }, + checkedAt, + models: [ + { + slug: input.config.model, + name: input.config.model, + isCustom: false, + isDefault: true, + capabilities: { + optionDescriptors: + input.config.model === "deepseek/deepseek-v4.1-flash" + ? [ + { + id: "variant", + label: "Reasoning", + type: "select", + options: [{ id: "low", label: "Low", isDefault: true }], + }, + ] + : [], + }, + }, + ], + probe: { + installed: true, + version: null, + status: configured ? "ready" : "warning", + auth: { status: "authenticated", profileId: credentials.accountId }, + message: configured + ? "Kilo Cloud uses the configured remote repository and Kilo credit. Only Full access is supported; cloud tools and subagents cannot be restricted. No local files are uploaded. Model availability and remote runtime version are unverified until a task runs." + : "Choose a GitHub repository and allow paid cloud execution before starting a task.", + }, + }), + instanceId: input.instanceId, + driver: kind, + displayName: input.displayName ?? "Kilo Cloud", + continuation: { groupKey: continuationKey }, + }; + const authenticatedSnapshot = snapshot; + const unavailable = ( + operation: "commit-message" | "pr-content" | "branch-name" | "thread-title", + ) => + Effect.fail( + new TextGenerationError({ + operation, + detail: + "Kilo Cloud does not generate local workspace metadata or start background paid tasks.", + }), + ); + return { + instanceId: input.instanceId, + driverKind: kind, + displayName: input.displayName, + accentColor: input.accentColor, + enabled: input.enabled, + continuationIdentity: { driverKind: kind, continuationKey }, + orchestrationAdapter: adapter, + textGeneration: { + generateCommitMessage: () => unavailable("commit-message"), + generatePrContent: () => unavailable("pr-content"), + generateBranchName: () => unavailable("branch-name"), + generateThreadTitle: () => unavailable("thread-title"), + }, + snapshot: { + getSnapshot: Effect.sync(() => snapshot), + refresh: Effect.gen(function* () { + const current = yield* account.load.pipe(Effect.orElseSucceed(() => null)); + if (current?.accountId === credentials.accountId) snapshot = authenticatedSnapshot; + else { + snapshot = { + ...snapshot, + status: "error", + message: + "Kilo login changed or is unavailable. Reconfigure this cloud account; existing remote tasks may still be running.", + }; + } + yield* PubSub.publish(changes, snapshot); + return snapshot; + }), + streamChanges: Stream.fromPubSub(changes), + applyUsageLimits: () => Effect.void, + resolveMaintenance: () => + Effect.succeed( + makeManualOnlyProviderMaintenanceCapabilities({ + provider: kind, + packageName: "@kilocode/cli", + }), + ), + }, + }; + }).pipe( + Effect.mapError( + (cause) => + new ProviderDriverError({ + driver: kind, + instanceId: input.instanceId, + detail: + "Kilo Cloud could not initialize. Use an official Kilo login in the selected profile.", + cause, + }), + ), + ), +}; diff --git a/apps/server/src/provider/builtInDrivers.ts b/apps/server/src/provider/builtInDrivers.ts index c82cfd564322..d783b74c3f49 100644 --- a/apps/server/src/provider/builtInDrivers.ts +++ b/apps/server/src/provider/builtInDrivers.ts @@ -27,6 +27,7 @@ import { CodexDriver, type CodexDriverEnv } from "./Drivers/CodexDriver.ts"; import { CursorDriver, type CursorDriverEnv } from "./Drivers/CursorDriver.ts"; import { GrokDriver, type GrokDriverEnv } from "./Drivers/GrokDriver.ts"; import { OpenCodeDriver, type OpenCodeDriverEnv } from "./Drivers/OpenCodeDriver.ts"; +import { KiloCloudDriver, type KiloCloudDriverEnv } from "./Drivers/KiloCloudDriver.ts"; import { KiloDriver, type KiloDriverEnv } from "./Drivers/KiloDriver.ts"; import { PiDriver, type PiDriverEnv } from "./Drivers/PiDriver.ts"; import type { AnyProviderDriver } from "./ProviderDriver.ts"; @@ -45,7 +46,8 @@ export type BuiltInDriversEnv = | GrokDriverEnv | OpenCodeDriverEnv | PiDriverEnv - | KiloDriverEnv; + | KiloDriverEnv + | KiloCloudDriverEnv; /** * Ordered list of built-in drivers. Order matters only for tie-breaking in @@ -59,6 +61,7 @@ export const BUILT_IN_DRIVERS: ReadonlyArray; accountId: string } | undefined; + const load = Effect.gen(function* () { + const saved = yield* decodeAuth( + yield* fs.readFileString(path.join(profileDirectory, "data", "kilo", "auth.json")), + ); + const token = saved.kilo.type === "oauth" ? saved.kilo.access : saved.kilo.key; + if (cached && Redacted.value(cached.token) === token) return cached; + const client = yield* HttpClient.HttpClient; + const response = yield* client.execute( + HttpClientRequest.get("https://app.kilo.ai/api/profile", { + headers: { authorization: `Bearer ${token}` }, + }), + ); + if (response.status !== 200) + return yield* new KiloCloudError({ operation: "authentication", reason: "rejected" }); + const profile = yield* decodeProfile(yield* response.json); + if (!profile.hasPersonalAccount) + return yield* new KiloCloudError({ operation: "personal-account", reason: "unsupported" }); + cached = { accountId: profile.user.id, token: Redacted.make(token) }; + return cached; + }).pipe( + Effect.scoped, + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.provide(FetchHttpClient.layer), + Effect.timeout("15 seconds"), + Effect.mapError(() => new KiloCloudError({ operation: "authentication", reason: "rejected" })), + ); + return { load }; +}); diff --git a/apps/server/src/provider/kilo/KiloCloudJournal.ts b/apps/server/src/provider/kilo/KiloCloudJournal.ts new file mode 100644 index 000000000000..24ca12c08afa --- /dev/null +++ b/apps/server/src/provider/kilo/KiloCloudJournal.ts @@ -0,0 +1,126 @@ +import { + OrchestrationV2ProviderThread, + OrchestrationV2ProviderTurn, + PositiveInt, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import { CloudBinding } from "./KiloCloudWebClient.ts"; + +export const CloudIntent = Schema.Struct({ + revision: Schema.Number, + accountId: Schema.NonEmptyString, + repository: Schema.NonEmptyString, + branch: Schema.NonEmptyString, + operationKey: Schema.NonEmptyString, + messageId: Schema.NonEmptyString, + payloadHash: Schema.NonEmptyString, + policyHash: Schema.NonEmptyString, + binding: Schema.NullOr(CloudBinding), + prepared: Schema.NullOr( + Schema.Struct({ + cloudAgentSessionId: Schema.NonEmptyString, + kiloSessionId: Schema.NonEmptyString, + }), + ), + state: Schema.Literals(["admission_unknown", "active", "completed", "failed", "interrupted"]), + interruptRequested: Schema.Boolean, + answeredRequestIds: Schema.Array(Schema.String), + providerThread: OrchestrationV2ProviderThread, + providerTurn: OrchestrationV2ProviderTurn, + runOrdinal: PositiveInt, +}); +export type CloudIntent = typeof CloudIntent.Type; +export class CloudJournalError extends Schema.TaggedError()( + "CloudJournalError", + { + operation: Schema.Literals(["read", "write"]), + cause: Schema.optional(Schema.Defect()), + }, +) {} +const codec = Schema.fromJsonString(Schema.toCodecJson(CloudIntent)); +const encode = Schema.encodeEffect(codec); +const decode = Schema.decodeUnknownEffect(codec); + +/** A FULL-synchronous SQLite commit precedes paid admission. This also serializes + * old/new drivers during settings reload and works without platform-specific + * directory fsync. Records contain correlations, never credentials or prompts. + */ +export const make = Effect.fn("KiloCloudJournal.make")(function* (directory: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fs + .makeDirectory(directory, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => new CloudJournalError({ operation: "write", cause }))); + const filename = path.join(directory, "intents.sqlite"); + const context = yield* Layer.build(NodeSqliteClient.layer({ filename })).pipe( + Effect.mapError((cause) => new CloudJournalError({ operation: "write", cause })), + ); + const sql = Context.get(context, SqlClient.SqlClient); + yield* fs + .chmod(filename, 0o600) + .pipe(Effect.mapError((cause) => new CloudJournalError({ operation: "write", cause }))); + yield* Effect.gen(function* () { + yield* sql`PRAGMA busy_timeout = 5000`; + yield* sql`PRAGMA synchronous = FULL`; + yield* sql`CREATE TABLE IF NOT EXISTS intents (operation_key TEXT PRIMARY KEY, thread_id TEXT NOT NULL, state TEXT NOT NULL, body TEXT NOT NULL)`; + yield* sql`CREATE INDEX IF NOT EXISTS cloud_intents_thread ON intents(thread_id)`; + yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS one_active_cloud_intent ON intents(thread_id) WHERE state IN ('active', 'admission_unknown')`; + }).pipe(Effect.mapError((cause) => new CloudJournalError({ operation: "write", cause }))); + const read = Effect.gen(function* () { + const rows = yield* sql<{ body: string }>`SELECT body FROM intents ORDER BY rowid`; + return yield* Effect.forEach(rows, (row) => decode(row.body)); + }).pipe(Effect.mapError((cause) => new CloudJournalError({ operation: "read", cause }))); + return { + read, + readThread: (threadId: string) => + Effect.gen(function* () { + const rows = yield* sql<{ + body: string; + }>`SELECT body FROM intents WHERE thread_id = ${threadId} ORDER BY rowid`; + return yield* Effect.forEach(rows, (row) => decode(row.body)); + }).pipe(Effect.mapError((cause) => new CloudJournalError({ operation: "read", cause }))), + reserve: (intent: CloudIntent) => + Effect.gen(function* () { + const body = yield* encode(intent); + const rows = + yield* sql`INSERT INTO intents (operation_key, thread_id, state, body) VALUES (${intent.operationKey}, ${intent.providerThread.id}, ${intent.state}, ${body}) ON CONFLICT DO NOTHING RETURNING operation_key`; + return rows.length === 1; + }).pipe(Effect.mapError((cause) => new CloudJournalError({ operation: "write", cause }))), + save: (intent: CloudIntent) => + Effect.gen(function* () { + const rows = yield* sql<{ + body: string; + }>`SELECT body FROM intents WHERE operation_key = ${intent.operationKey}`; + const row = rows[0]; + if (!row) return yield* new CloudJournalError({ operation: "write" }); + const prior = yield* decode(row.body); + if ( + prior.revision !== intent.revision || + prior.accountId !== intent.accountId || + prior.repository !== intent.repository || + prior.branch !== intent.branch || + prior.messageId !== intent.messageId || + prior.payloadHash !== intent.payloadHash || + prior.policyHash !== intent.policyHash || + prior.providerThread.id !== intent.providerThread.id || + prior.providerTurn.id !== intent.providerTurn.id || + (["completed", "failed", "interrupted"].includes(prior.state) && + prior.state !== intent.state) + ) + return yield* new CloudJournalError({ operation: "write" }); + const next = { ...intent, revision: intent.revision + 1 }; + const body = yield* encode(next); + const updated = + yield* sql`UPDATE intents SET state = ${intent.state}, body = ${body} WHERE operation_key = ${intent.operationKey} AND body = ${row.body} RETURNING operation_key`; + if (updated.length !== 1) return yield* new CloudJournalError({ operation: "write" }); + return next; + }).pipe(Effect.mapError((cause) => new CloudJournalError({ operation: "write", cause }))), + }; +}); diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.live.test.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.live.test.ts new file mode 100644 index 000000000000..74b4db8b602a --- /dev/null +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.live.test.ts @@ -0,0 +1,44 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { describe } from "vite-plus/test"; +import * as Effect from "effect/Effect"; +import * as Account from "./KiloCloudAccount.ts"; +import * as Cloud from "./KiloCloudWebClient.ts"; + +// Read-only opt-in. This never prepares a sandbox, submits a prompt or answers a request. +const profile = process.env.KILO_CLOUD_TEST_PROFILE; +const sessionId = process.env.KILO_CLOUD_READ_SESSION; +describe.skipIf(!profile || !sessionId)("Kilo Cloud customer read contract", () => { + it.live( + "reads native history and independent sandbox/billing evidence with official CLI login", + () => + Effect.gen(function* () { + const account = yield* Account.make(profile!); + const credentials = yield* account.load; + const client = Cloud.make({ ...credentials, credentials: account.load }); + const session = yield* client.getSession(sessionId!); + const binding: Cloud.CloudBinding = { + accountId: credentials.accountId, + cloudAgentSessionId: session.sessionId, + kiloSessionId: session.kiloSessionId, + worktreeId: session.worktreeId, + repository: session.githubRepo, + branch: session.upstreamBranch ?? "main", + }; + const page = yield* client.history(binding); + assert.equal(page.kiloSessionId, session.kiloSessionId); + assert.isTrue( + page.history?.messages.some( + (message) => + message.info.role === "assistant" && + message.parts.some((part) => part.type === "text" && !!part.text), + ) ?? false, + ); + const sandbox = yield* client.sandbox(binding); + const billing = yield* client.billing(binding); + assert.isTrue(sandbox.observedAt > 0); + assert.isTrue(["session", "payer_shared"].includes(billing.attribution)); + }).pipe(Effect.provide(NodeServices.layer)), + 30_000, + ); +}); diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts new file mode 100644 index 000000000000..8bd19b50f68d --- /dev/null +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts @@ -0,0 +1,343 @@ +// @effect-diagnostics nodeBuiltinImport:off - exercises customer HTTP semantics over real sockets. +import * as NodeHttp from "node:http"; +import * as NodeEvents from "node:events"; +import { NodeWS } from "@effect/platform-node/NodeSocket"; +import * as Stream from "effect/Stream"; +import { afterEach, describe, expect, it } from "vite-plus/test"; +import * as Effect from "effect/Effect"; +import * as Clock from "effect/Clock"; +import * as Redacted from "effect/Redacted"; +import * as Cloud from "./KiloCloudWebClient.ts"; + +const cleanups: Array<() => Promise> = []; +afterEach(async () => { + for (const cleanup of cleanups.splice(0)) await cleanup(); +}); +const binding: Cloud.CloudBinding = { + accountId: "customer-a", + cloudAgentSessionId: "workspace_12345678-1234-1234-1234-123456789abc", + worktreeId: "worktree_12345678-1234-1234-1234-123456789abc", + kiloSessionId: "ses_synthetic", + repository: "fixture/project", + branch: "main", +}; +const messageId = "msg_0123456789ab0123456789ABCD"; +const start = { + operationKey: "12345678-1234-4234-9234-123456789abc", + initialMessageId: messageId, + prompt: "Read README only", + repository: binding.repository, + branch: "main", + model: "fixture/model", +}; +const session = { + sessionId: binding.cloudAgentSessionId, + kiloSessionId: binding.kiloSessionId, + worktreeId: binding.worktreeId, + userId: binding.accountId, + githubRepo: binding.repository, + upstreamBranch: "main", + autoCommit: false, + initialMessageId: messageId, + execution: null, +}; +const run = Effect.runPromise; +async function server( + handler: (request: NodeHttp.IncomingMessage, response: NodeHttp.ServerResponse) => void, + profiles: unknown = [], + bindings: unknown = [], +) { + const server = NodeHttp.createServer((request, response) => { + if (request.url?.startsWith("/api/trpc/agentProfiles.listRepoBindings")) + return json(response, bindings); + if (request.url?.startsWith("/api/trpc/agentProfiles.list")) return json(response, profiles); + handler(request, response); + }); + server.listen(0, "127.0.0.1"); + await NodeEvents.EventEmitter.once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing fixture address"); + cleanups.push(async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }); + const options = { + accountId: binding.accountId, + token: Redacted.make("fixture-token"), + origin: `http://127.0.0.1:${address.port}`, + }; + return { options, client: Cloud.make(options), httpServer: server }; +} +function json(response: NodeHttp.ServerResponse, data: unknown) { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ result: { data } })); +} +describe("Kilo personal Cloud control-plane customer API", () => { + it("authenticates a customer WebSocket, resumes its cursor and rejects a foreign session event", async () => { + const expiresAt = await run(Clock.currentTimeMillis); + const { client, httpServer } = await server((req, res) => { + expect(req.url).toBe("/api/cloud-agent-next/sessions/stream-ticket"); + expect(req.headers.authorization).toBe("Bearer fixture-token"); + req.resume(); + req.on("end", () => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ ticket: "single-use-fixture", expiresAt: expiresAt + 60000 })); + }); + }); + const sockets = new NodeWS.WebSocketServer({ noServer: true }); + cleanups.unshift(async () => { + for (const socket of sockets.clients) socket.terminate(); + await new Promise((resolve) => sockets.close(() => resolve())); + }); + httpServer.on("upgrade", (req, socket, head) => { + const url = new URL(req.url!, "http://localhost"); + expect(url.searchParams.get("fromId")).toBe("37"); + expect(url.searchParams.get("ticket")).toBe("single-use-fixture"); + expect(url.searchParams.get("cloudAgentSessionId")).toBe(binding.cloudAgentSessionId); + sockets.handleUpgrade(req, socket, head, (connection) => { + connection.send( + JSON.stringify({ + eventId: 38, + sessionId: binding.cloudAgentSessionId, + streamEventType: "cloud.message.completed", + data: { messageId }, + }), + ); + connection.send( + JSON.stringify({ + eventId: 39, + sessionId: "workspace_00000000-0000-0000-0000-000000000000", + streamEventType: "cloud.message.failed", + data: { messageId }, + }), + ); + }); + }); + const received: number[] = []; + const error = await run( + client.events(binding, 37).pipe( + Stream.tap((event) => + Effect.sync(() => { + received.push(event.eventId); + }), + ), + Stream.runDrain, + Effect.flip, + ), + ); + expect(received).toEqual([38]); + expect(error.operation).toBe("events"); + }); + it.each([ + "varCount", + "commandCount", + "mcpServerCount", + "skillCount", + "agentCount", + "kiloCommandCount", + ])("does not admit a paid task when an inherited profile has %s", async (counter) => { + let mutations = 0; + const profile = { + id: "profile", + isDefault: true, + varCount: 0, + commandCount: 0, + mcpServerCount: 0, + skillCount: 0, + agentCount: 0, + kiloCommandCount: 0, + [counter]: 1, + }; + const { client } = await server( + (_request, response) => { + mutations++; + response.end(); + }, + [profile], + ); + expect((await run(client.prepare(start).pipe(Effect.flip))).reason).toBe("rejected"); + expect(mutations).toBe(0); + }); + it("fails closed when the repository binding references an unavailable profile", async () => { + let mutations = 0; + const { client } = await server( + (_request, response) => { + mutations++; + response.end(); + }, + [], + [{ repoFullName: "FIXTURE/PROJECT", platform: "github", profileId: "unavailable" }], + ); + expect((await run(client.prepare(start).pipe(Effect.flip))).reason).toBe("rejected"); + expect(mutations).toBe(0); + }); + it("uses customer authentication and fixed admission identities, no commits, setup or local data", async () => { + const bodies: Record[] = []; + const { client } = await server((req, res) => { + expect(req.headers.authorization).toBe("Bearer fixture-token"); + expect(req.url).toBe("/api/trpc/cloudAgentNext.prepareSession"); + let body = ""; + req.on("data", (chunk) => { + body += String(chunk); + }); + req.on("end", () => { + bodies.push(JSON.parse(body)); + json(res, { + cloudAgentSessionId: binding.cloudAgentSessionId, + kiloSessionId: binding.kiloSessionId, + }); + }); + }); + await run(client.prepare(start)); + expect(bodies).toHaveLength(1); + expect(bodies[0]).toMatchObject({ + operationKey: start.operationKey, + initialMessageId: messageId, + githubRepo: binding.repository, + upstreamBranch: "main", + autoCommit: false, + autoInitiate: true, + envVars: {}, + setupCommands: [], + mcpServers: {}, + runtimeSkills: [], + runtimeAgents: [], + mode: "code", + }); + expect(bodies[0]).not.toHaveProperty("attachments"); + }); + it("leaves acceptance uncertain and never retries a socket lost after paid admission", async () => { + let accepted = 0; + const { client } = await server((req, res) => { + req.resume(); + req.on("end", () => { + accepted++; + res.destroy(); + }); + }); + const error = await run(client.prepare(start).pipe(Effect.flip)); + expect(error.reason).toBe("admission_unknown"); + expect(error.messageId).toBe(messageId); + expect(accepted).toBe(1); + }); + it.each(["userId", "githubRepo", "kiloSessionId", "worktreeId", "autoCommit"])( + "prevents a follow-up on changed %s", + async (field) => { + let sends = 0; + const changed = { + ...session, + [field]: + field === "autoCommit" + ? true + : field === "worktreeId" + ? "worktree_aaaaaaaa-1234-1234-1234-123456789abc" + : field === "kiloSessionId" + ? "ses_other" + : "other", + }; + const { client } = await server((req, res) => { + if (req.method === "POST") sends++; + json(res, changed); + }); + expect( + ( + await run( + client + .send(binding, { messageId, prompt: "Continue", model: "fixture/model" }) + .pipe(Effect.flip), + ) + ).reason, + ).toBe("wrong_owner"); + expect(sends).toBe(0); + }, + ); + it("rejects an account switch before any HTTP request, while allowing same-account token refresh", async () => { + let requests = 0; + const { options } = await server((req, res) => { + requests++; + expect(req.headers.authorization).toBe("Bearer refreshed"); + json(res, session); + }); + const refreshed = Cloud.make({ + ...options, + credentials: Effect.succeed({ + accountId: binding.accountId, + token: Redacted.make("refreshed"), + }), + }); + await run(refreshed.getSession(binding.cloudAgentSessionId)); + const changed = Cloud.make({ + ...options, + credentials: Effect.succeed({ accountId: "customer-b", token: Redacted.make("other") }), + }); + expect( + (await run(changed.getSession(binding.cloudAgentSessionId).pipe(Effect.flip))).reason, + ).toBe("wrong_owner"); + expect(requests).toBe(1); + }); + it("rejects a transcript with a part from another session", async () => { + const { client } = await server((_req, res) => + json(res, { + kiloSessionId: binding.kiloSessionId, + watermarkEventId: 84, + history: { + nextCursor: null, + omittedItemCount: 0, + messages: [ + { + info: { + id: "assistant", + sessionID: binding.kiloSessionId, + role: "assistant", + parentID: messageId, + time: { created: 1, completed: 2 }, + }, + parts: [ + { + id: "part", + messageID: "assistant", + sessionID: "ses_other", + type: "text", + text: "Not this account", + }, + ], + }, + ], + }, + }), + ); + expect((await run(client.history(binding).pipe(Effect.flip))).reason).toBe("wrong_owner"); + }); + it("keeps interruption acceptance, task terminality, sandbox sleep and shared-payer billing separate", async () => { + const { client } = await server((req, res) => { + if (req.url?.includes("getSession?")) json(res, session); + else if (req.url?.includes("interruptSession")) json(res, { success: true }); + else if (req.url?.includes("getMessageResult")) { + const input = JSON.parse(new URL(req.url, "http://localhost").searchParams.get("input")!); + expect(input.expectedWorktreeId).toBe(binding.worktreeId); + json(res, { + cloudAgentSessionId: binding.cloudAgentSessionId, + messageId, + status: "interrupted", + }); + } else if (req.url?.includes("getSandboxStatus")) + json(res, { + status: "active", + observedAt: 1, + inactivityTimeoutMs: 600000, + estimatedSleepAt: null, + }); + else + json(res, { + phase: "active", + attribution: "payer_shared", + estimatedHourlyRateMicrodollars: 1203120, + estimatedIntervalAmountMicrodollars: null, + }); + }); + expect(await run(client.interrupt(binding))).toEqual({ success: true }); + expect((await run(client.result(binding, messageId)))?.status).toBe("interrupted"); + expect((await run(client.sandbox(binding))).status).toBe("active"); + expect((await run(client.billing(binding))).phase).toBe("active"); + }); +}); diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.ts new file mode 100644 index 000000000000..9964ae9adc57 --- /dev/null +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.ts @@ -0,0 +1,605 @@ +import * as Effect from "effect/Effect"; +import * as NodeSocket from "@effect/platform-node/NodeSocket"; +import * as Socket from "effect/unstable/socket/Socket"; +import * as Redacted from "effect/Redacted"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { KiloCloudError } from "./KiloCloudClient.ts"; + +// Customer routes used by Kilo's web/mobile clients (Kilo-Org/cloud 78ea0a5e). +// workspace_* is the control-plane session; ses_* is its conversation; worktree_* +// owns the remote files. Numeric `version` fields are not protocol versions. +const CloudId = Schema.String.check(Schema.isPattern(/^workspace_[0-9a-f-]{36}$/i)); +const WorktreeId = Schema.String.check(Schema.isPattern(/^worktree_[0-9a-f-]{36}$/i)); +const NativeId = Schema.String.check(Schema.isPattern(/^ses_[0-9A-Za-z]+$/)); +const MessageId = Schema.String.check(Schema.isPattern(/^msg_[0-9a-f]{12}[0-9A-Za-z]{14}$/)); +export const CloudBinding = Schema.Struct({ + accountId: Schema.NonEmptyString, + cloudAgentSessionId: CloudId, + kiloSessionId: NativeId, + worktreeId: WorktreeId, + repository: Schema.NonEmptyString, + branch: Schema.NonEmptyString, +}); +export type CloudBinding = typeof CloudBinding.Type; +const Prepared = Schema.Struct({ cloudAgentSessionId: CloudId, kiloSessionId: NativeId }); +const Sent = Schema.Struct({ + cloudAgentSessionId: CloudId, + messageId: MessageId, + status: Schema.Literal("started"), + delivery: Schema.Literals(["sent", "queued"]), +}); +const Session = Schema.Struct({ + sessionId: CloudId, + kiloSessionId: NativeId, + userId: Schema.NonEmptyString, + orgId: Schema.optional(Schema.String), + worktreeId: WorktreeId, + githubRepo: Schema.NonEmptyString, + upstreamBranch: Schema.optional(Schema.String), + autoCommit: Schema.Boolean, + initialMessageId: Schema.optional(MessageId), + execution: Schema.NullOr(Schema.Record(Schema.String, Schema.Unknown)), +}); +const Result = Schema.NullOr( + Schema.Struct({ + cloudAgentSessionId: CloudId, + messageId: MessageId, + status: Schema.Literals(["queued", "running", "completed", "failed", "interrupted"]), + }), +); +const NativeMessage = Schema.Struct({ + info: Schema.Struct({ + id: Schema.NonEmptyString, + sessionID: NativeId, + role: Schema.Literals(["user", "assistant"]), + parentID: Schema.optional(Schema.String), + time: Schema.Struct({ created: Schema.Number, completed: Schema.optional(Schema.Number) }), + cost: Schema.optional(Schema.Number), + finish: Schema.optional(Schema.String), + }), + parts: Schema.Array( + Schema.Struct({ + id: Schema.NonEmptyString, + sessionID: NativeId, + messageID: Schema.NonEmptyString, + type: Schema.String, + text: Schema.optional(Schema.String), + tool: Schema.optional(Schema.String), + callID: Schema.optional(Schema.String), + state: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + }), + ), +}); +export type CloudMessage = typeof NativeMessage.Type; +const History = Schema.Struct({ + kiloSessionId: NativeId, + history: Schema.NullOr( + Schema.Struct({ + messages: Schema.Array(NativeMessage), + nextCursor: Schema.NullOr(Schema.String), + omittedItemCount: Schema.Number, + }), + ), + watermarkEventId: Schema.NullOr(Schema.Number), +}); +const Sandbox = Schema.Struct({ + status: Schema.Literals([ + "active", + "sleeping", + "starting", + "stopping", + "error", + "unreachable", + "unknown", + ]), + observedAt: Schema.Number, + inactivityTimeoutMs: Schema.NullOr(Schema.Number), + estimatedSleepAt: Schema.NullOr(Schema.Number), +}); +const Billing = Schema.Struct({ + phase: Schema.Literals(["idle", "active", "stopping", "settling", "unavailable"]), + attribution: Schema.Literals(["payer_shared", "session"]), + estimatedHourlyRateMicrodollars: Schema.NullOr(Schema.Number), + estimatedIntervalAmountMicrodollars: Schema.NullOr(Schema.Number), +}); +const Count = Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)); +const Profiles = Schema.Array( + Schema.Struct({ + id: Schema.NonEmptyString, + isDefault: Schema.Boolean, + varCount: Count, + commandCount: Count, + mcpServerCount: Count, + skillCount: Count, + agentCount: Count, + kiloCommandCount: Count, + }), +); +const Bindings = Schema.Array( + Schema.Struct({ + repoFullName: Schema.NonEmptyString, + platform: Schema.String, + profileId: Schema.NonEmptyString, + }), +); +const Permission = Schema.Struct({ + id: Schema.NonEmptyString, + sessionID: NativeId, + permission: Schema.String, + patterns: Schema.Array(Schema.String), +}); +const Question = Schema.Struct({ + id: Schema.NonEmptyString, + sessionID: NativeId, + questions: Schema.Array( + Schema.Struct({ + header: Schema.String, + question: Schema.String, + options: Schema.Array(Schema.Struct({ label: Schema.String, description: Schema.String })), + multiple: Schema.optional(Schema.Boolean), + custom: Schema.optional(Schema.Boolean), + }), + ), +}); +export type CloudInteraction = typeof Permission.Type | typeof Question.Type; +const Pending = Schema.Struct({ + questions: Schema.Array(Question), + permissions: Schema.Array(Permission), +}); +const Acknowledged = Schema.Struct({ success: Schema.Boolean }); +const envelope = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Struct({ result: Schema.Struct({ data: Schema.Unknown }) })), +); +const Ticket = Schema.Struct({ ticket: Schema.NonEmptyString, expiresAt: Schema.Number }); +const StreamEvent = Schema.Struct({ + eventId: Schema.Number, + sessionId: CloudId, + streamEventType: Schema.NonEmptyString, + data: Schema.Record(Schema.String, Schema.Unknown), +}); +const decodeStreamEvent = Schema.decodeUnknownEffect(Schema.fromJsonString(StreamEvent)); +const decodeJson = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown)); +const isCloudError = Schema.is(KiloCloudError); +const encode = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); + +export const make = (options: { + readonly token: Redacted.Redacted; + /** Immutable authenticated personal account id, never a display name or token hash. */ + readonly accountId: string; + readonly origin?: string; + readonly credentials?: Effect.Effect< + { readonly token: Redacted.Redacted; readonly accountId: string }, + KiloCloudError + >; +}) => { + const origin = options.origin ?? "https://app.kilo.ai"; + const allowed = origin === "https://app.kilo.ai" || /^http:\/\/127\.0\.0\.1:\d+$/.test(origin); + const failure = (operation: string, reason: KiloCloudError["reason"], messageId?: string) => + new KiloCloudError({ operation, reason, ...(messageId ? { messageId } : {}) }); + const request = ( + operation: string, + body: unknown, + schema: Schema.Decoder, + mutation = false, + messageId?: string, + ) => { + const uncertain = () => + failure(operation, mutation ? "admission_unknown" : "invalid_response", messageId); + if (!allowed) return Effect.fail(failure(operation, "rejected")); + return Effect.gen(function* () { + const credentials = options.credentials ? yield* options.credentials : options; + if (credentials.accountId !== options.accountId) + return yield* failure(operation, "wrong_owner"); + const client = yield* HttpClient.HttpClient; + const req = HttpClientRequest.make(mutation ? "POST" : "GET")( + operation === "stream-ticket" + ? `${origin}/api/cloud-agent-next/sessions/stream-ticket` + : `${origin}/api/trpc/${operation}${mutation ? "" : `?input=${encodeURIComponent(encode(body))}`}`, + { + headers: { + authorization: `Bearer ${Redacted.value(credentials.token)}`, + "content-type": "application/json", + }, + }, + ); + const response = yield* client.execute( + mutation ? HttpClientRequest.bodyText(req, encode(body), "application/json") : req, + ); + if (response.status < 200 || response.status >= 300) + return yield* response.status >= 500 || response.status === 408 || response.status === 409 + ? uncertain() + : failure(operation, "rejected", messageId); + const chunks: Uint8Array[] = []; + let size = 0; + yield* response.stream.pipe( + Stream.runForEach((chunk) => { + size += chunk.byteLength; + if (size > 8 * 1024 * 1024) return Effect.fail(uncertain()); + chunks.push(chunk); + return Effect.void; + }), + ); + const text = Buffer.concat(chunks).toString("utf8"); + const data = + operation === "stream-ticket" + ? yield* decodeJson(text) + : (yield* envelope(text)).result.data; + return yield* Schema.decodeUnknownEffect(schema)(data); + }).pipe( + Effect.scoped, + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.provide(FetchHttpClient.layer), + Effect.timeout("30 seconds"), + Effect.mapError((error) => (isCloudError(error) ? error : uncertain())), + ); + }; + const own = (binding: CloudBinding) => + binding.accountId === options.accountId + ? Effect.void + : Effect.fail(failure("ownership", "wrong_owner")); + const getSession = (id: string) => + request("cloudAgentNext.getSession", { cloudAgentSessionId: id }, Session).pipe( + Effect.flatMap((session) => + session.sessionId === id && + session.userId === options.accountId && + session.orgId === undefined + ? Effect.succeed(session) + : Effect.fail(failure("getSession", "wrong_owner")), + ), + ); + const check = (binding: CloudBinding) => + own(binding).pipe( + Effect.andThen(getSession(binding.cloudAgentSessionId)), + Effect.flatMap((session) => + session.kiloSessionId === binding.kiloSessionId && + session.worktreeId === binding.worktreeId && + session.githubRepo === binding.repository && + session.upstreamBranch === binding.branch && + !session.autoCommit + ? Effect.succeed(session) + : Effect.fail(failure("ownership", "wrong_owner")), + ), + ); + // The web API merges the default and repository profiles additively. Empty + // overrides do not disable them. Read only summaries, never profile secrets. + const preflight = (repository: string) => + Effect.gen(function* () { + const profiles = yield* request("agentProfiles.list", {}, Profiles); + const bindings = yield* request("agentProfiles.listRepoBindings", {}, Bindings); + const defaults = profiles.filter((profile) => profile.isDefault); + const relevant = bindings.filter( + (binding) => + binding.platform === "github" && + binding.repoFullName.toLowerCase() === repository.toLowerCase(), + ); + if ( + defaults.length > 1 || + relevant.length > 1 || + new Set(profiles.map((profile) => profile.id)).size !== profiles.length + ) + return yield* failure("profile-preflight", "rejected"); + const ids = new Set([ + ...defaults.map((profile) => profile.id), + ...relevant.map((binding) => binding.profileId), + ]); + for (const id of ids) { + const profile = profiles.find((profile) => profile.id === id); + if ( + !profile || + [ + profile.varCount, + profile.commandCount, + profile.mcpServerCount, + profile.skillCount, + profile.agentCount, + profile.kiloCommandCount, + ].some((count) => count !== 0) + ) + return yield* failure("profile-preflight", "rejected"); + } + }); + return { + getSession, + /** Short-lived customer tickets only. A closed socket has no stop semantics. */ + events: (binding: CloudBinding, fromId = 0) => + Stream.unwrap( + Effect.gen(function* () { + yield* own(binding); + const ticket = yield* request( + "stream-ticket", + { cloudAgentSessionId: binding.cloudAgentSessionId }, + Ticket, + true, + ); + const socketOrigin = + origin === "https://app.kilo.ai" + ? "wss://cloud-agent-next.kilosessions.ai" + : origin.replace("http:", "ws:"); + const url = new URL(`${socketOrigin}/stream`); + url.searchParams.set("cloudAgentSessionId", binding.cloudAgentSessionId); + url.searchParams.set("ticket", ticket.ticket); + url.searchParams.set("fromId", String(fromId)); + const socket = yield* Socket.makeWebSocket(url.toString(), { + openTimeout: "10 seconds", + }).pipe(Effect.provide(NodeSocket.layerWebSocketConstructor)); + return Stream.fromPull(Socket.readerString(socket)).pipe( + Stream.mapEffect((text) => + Effect.gen(function* () { + if (text.length > 8 * 1024 * 1024) + return yield* failure("events", "invalid_response"); + const credentials = options.credentials ? yield* options.credentials : options; + if (credentials.accountId !== binding.accountId) + return yield* failure("events", "wrong_owner"); + const event = yield* decodeStreamEvent(text); + if (event.sessionId !== binding.cloudAgentSessionId) + return yield* failure("events", "wrong_owner"); + return event; + }), + ), + ); + }), + ).pipe( + Stream.scoped, + Stream.mapError(() => failure("events", "invalid_response")), + ), + // Recovery only: enumerate access-checked metadata and correlate the persisted + // initial message, never replay a paid prepare after a lost response. + findAdmission: (repository: string, initialMessageId: string) => + Effect.gen(function* () { + const page = yield* request( + "cliSessionsV2.list", + { gitUrl: `https://github.com/${repository}`, limit: 100 }, + Schema.Struct({ + cliSessions: Schema.Array( + Schema.Struct({ + session_id: NativeId, + cloud_agent_session_id: Schema.NullOr(Schema.String), + }), + ), + }), + ); + const matches: Array = []; + for (const candidate of page.cliSessions) { + if (!candidate.cloud_agent_session_id?.startsWith("workspace_")) continue; + const session = yield* getSession(candidate.cloud_agent_session_id); + if ( + session.initialMessageId === initialMessageId && + session.githubRepo === repository && + session.kiloSessionId === candidate.session_id + ) + matches.push({ + cloudAgentSessionId: session.sessionId, + kiloSessionId: session.kiloSessionId, + }); + } + if (matches.length > 1) return yield* failure("reconcile-admission", "wrong_owner"); + return matches[0] ?? null; + }), + /** Admission is paid. Persist operationKey and initialMessageId before calling; no retries here. */ + prepare: (input: { + readonly operationKey: string; + readonly initialMessageId: string; + readonly prompt: string; + readonly repository: string; + readonly branch: string; + readonly model: string; + readonly variant?: string; + }) => + preflight(input.repository).pipe( + Effect.andThen( + request( + "cloudAgentNext.prepareSession", + { + operationKey: input.operationKey, + initialMessageId: input.initialMessageId, + prompt: input.prompt, + githubRepo: input.repository, + upstreamBranch: input.branch, + model: input.model, + ...(input.variant ? { variant: input.variant } : {}), + mode: "code", + autoInitiate: true, + autoCommit: false, + devcontainer: false, + // These are additive overrides. A separate profile preflight must reject inherited configuration before admission. + envVars: {}, + setupCommands: [], + mcpServers: {}, + runtimeSkills: [], + runtimeAgents: [], + }, + Prepared, + true, + input.initialMessageId, + ), + ), + ), + bind: ( + prepared: typeof Prepared.Type, + repository: string, + initialMessageId: string, + branch: string, + ) => + getSession(prepared.cloudAgentSessionId).pipe( + Effect.flatMap((session) => + session.kiloSessionId === prepared.kiloSessionId && + session.githubRepo === repository && + session.upstreamBranch === branch && + session.initialMessageId === initialMessageId && + !session.autoCommit + ? Effect.succeed({ + accountId: options.accountId, + cloudAgentSessionId: session.sessionId, + kiloSessionId: session.kiloSessionId, + worktreeId: session.worktreeId, + repository, + branch, + }) + : Effect.fail(failure("bind", "wrong_owner")), + ), + ), + send: ( + binding: CloudBinding, + input: { + readonly messageId: string; + readonly prompt: string; + readonly model: string; + readonly variant?: string; + }, + ) => + check(binding).pipe( + Effect.andThen( + request( + "cloudAgentNext.sendMessage", + { + cloudAgentSessionId: binding.cloudAgentSessionId, + expectedWorktreeId: binding.worktreeId, + messageId: input.messageId, + autoCommit: false, + payload: { + type: "prompt", + prompt: input.prompt, + mode: "code", + model: input.model, + ...(input.variant ? { variant: input.variant } : {}), + }, + }, + Sent, + true, + input.messageId, + ), + ), + Effect.flatMap((sent) => + sent.cloudAgentSessionId === binding.cloudAgentSessionId && + sent.messageId === input.messageId + ? Effect.asVoid(Effect.succeed(sent)) + : Effect.fail(failure("send", "admission_unknown", input.messageId)), + ), + ), + result: (binding: CloudBinding, messageId: string) => + own(binding).pipe( + Effect.andThen( + request( + "cloudAgentNext.getMessageResult", + { + cloudAgentSessionId: binding.cloudAgentSessionId, + expectedWorktreeId: binding.worktreeId, + messageId, + }, + Result, + ), + ), + Effect.flatMap((result) => + result === null || + (result.cloudAgentSessionId === binding.cloudAgentSessionId && + result.messageId === messageId) + ? Effect.succeed(result) + : Effect.fail(failure("result", "wrong_owner")), + ), + ), + history: (binding: CloudBinding, cursor?: string) => + own(binding).pipe( + Effect.andThen( + request( + "cliSessionsV2.getSessionMessagesPage", + { session_id: binding.kiloSessionId, limit: 50, ...(cursor ? { cursor } : {}) }, + History, + ), + ), + Effect.flatMap((page) => + page.kiloSessionId === binding.kiloSessionId && + (page.history === null || + page.history.messages.every( + (message) => + message.info.sessionID === binding.kiloSessionId && + message.parts.every( + (part) => + part.sessionID === binding.kiloSessionId && part.messageID === message.info.id, + ), + )) + ? Effect.succeed(page) + : Effect.fail(failure("history", "wrong_owner")), + ), + ), + pending: (binding: CloudBinding) => + check(binding).pipe( + Effect.andThen( + request( + "cloudAgentNext.getPendingInteractions", + { cloudAgentSessionId: binding.cloudAgentSessionId }, + Pending, + ), + ), + Effect.filterOrFail( + (pending) => + [...pending.questions, ...pending.permissions].every( + (request) => request.sessionID === binding.kiloSessionId, + ), + () => failure("pending", "wrong_owner"), + ), + ), + interrupt: (binding: CloudBinding) => + check(binding).pipe( + Effect.andThen( + request( + "cloudAgentNext.interruptSession", + { sessionId: binding.cloudAgentSessionId }, + Schema.Struct({ success: Schema.Boolean }), + true, + ), + ), + ), + replyPermission: ( + binding: CloudBinding, + permissionId: string, + response: "once" | "always" | "reject", + ) => + check(binding).pipe( + Effect.andThen( + request( + "cloudAgentNext.answerPermission", + { sessionId: binding.cloudAgentSessionId, permissionId, response }, + Acknowledged, + true, + ), + ), + ), + replyQuestion: ( + binding: CloudBinding, + questionId: string, + answers: ReadonlyArray>, + ) => + check(binding).pipe( + Effect.andThen( + request( + "cloudAgentNext.answerQuestion", + { sessionId: binding.cloudAgentSessionId, questionId, answers }, + Acknowledged, + true, + ), + ), + ), + sandbox: (binding: CloudBinding) => + own(binding).pipe( + Effect.andThen( + request( + "cloudAgentNext.getSandboxStatus", + { cloudAgentSessionId: binding.cloudAgentSessionId }, + Sandbox, + ), + ), + ), + billing: (binding: CloudBinding) => + own(binding).pipe( + Effect.andThen( + request( + "cloudAgentNext.getComputeBillingStatus", + { cloudAgentSessionId: binding.cloudAgentSessionId }, + Billing, + ), + ), + ), + }; +}; diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 8e105a49f37f..21d3155825bb 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -3131,8 +3131,15 @@ export default function ChatView(props: ChatViewProps) { const selectedProvider = selectedProviderEntry?.driverKind ?? requestedDriverKind; const activeProviderInstanceId = selectedProviderEntry?.instanceId ?? null; const activeProviderStatus = selectedProviderEntry?.snapshot ?? null; + const isCloudComposer = activeProviderStatus?.driver === "kilo-cloud"; + const persistedProviderThread = serverProjection?.providerThreads.find( + (thread) => thread.id === serverProjection.thread.activeProviderThreadId, + ); + const cloudExecution = persistedProviderThread?.nativeMetadata?.cloudExecution; + const isCloudThread = + !!cloudExecution || persistedProviderThread?.driver === "kilo-cloud" || isCloudComposer; const { enabled: interactionModeEnabled, interactionMode } = resolveComposerInteractionMode({ - planModeEnabled: settings.planModeEnabled, + planModeEnabled: settings.planModeEnabled && !isCloudComposer, provider: activeProviderStatus, interactionMode: composerInteractionMode ?? activeThread?.interactionMode ?? DEFAULT_INTERACTION_MODE, @@ -4137,7 +4144,7 @@ export default function ChatView(props: ChatViewProps) { [keybindings, terminalShortcutLabelOptions], ); const onToggleDiff = useCallback(() => { - if (!isServerThread) { + if (isCloudThread || !isServerThread) { return; } if (!diffOpen) { @@ -4146,7 +4153,7 @@ export default function ChatView(props: ChatViewProps) { if (activeThreadRef) { useRightPanelStore.getState().toggle(activeThreadRef, "diff"); } - }, [activeThreadRef, diffOpen, isServerThread, onDiffPanelOpen]); + }, [activeThreadRef, diffOpen, isCloudThread, isServerThread, onDiffPanelOpen]); const needsLoadBalancing = automaticEnvironment && !draftThread?.loadBalancedEnvironmentId; const loadBalancingCandidates = useMemo( @@ -4491,7 +4498,7 @@ export default function ChatView(props: ChatViewProps) { [activeThreadRef, storeSetTerminalOpen], ); const toggleTerminalVisibility = useCallback(() => { - if (!activeThreadRef) return; + if (isCloudThread || !activeThreadRef) return; const nextOpen = !terminalUiState.terminalOpen; if (nextOpen && terminalUiState.terminalIds.length === 0) { if (!activeThreadId || !activeProject) { @@ -4532,10 +4539,17 @@ export default function ChatView(props: ChatViewProps) { storeEnsureTerminal, terminalUiState.terminalIds.length, terminalUiState.terminalOpen, + isCloudThread, ]); const splitTerminal = useCallback( (direction: "horizontal" | "vertical" = "horizontal") => { - if (!activeThreadRef || hasReachedSplitLimit || !activeThreadId || !activeProject) { + if ( + isCloudThread || + !activeThreadRef || + hasReachedSplitLimit || + !activeThreadId || + !activeProject + ) { return; } const cwdForOpen = gitCwd ?? activeProject.workspaceRoot; @@ -4564,6 +4578,7 @@ export default function ChatView(props: ChatViewProps) { }); }, [ + isCloudThread, activeProject, activeThreadId, allocatableActiveTerminalIds, @@ -4578,7 +4593,7 @@ export default function ChatView(props: ChatViewProps) { ], ); const createNewTerminal = useCallback(() => { - if (!activeThreadRef || !activeThreadId || !activeProject) { + if (isCloudThread || !activeThreadRef || !activeThreadId || !activeProject) { return; } const cwdForOpen = gitCwd ?? activeProject.workspaceRoot; @@ -4602,6 +4617,7 @@ export default function ChatView(props: ChatViewProps) { }, }); }, [ + isCloudThread, activeProject, activeThreadId, allocatableActiveTerminalIds, @@ -4656,7 +4672,7 @@ export default function ChatView(props: ChatViewProps) { rememberAsLastInvoked?: boolean; }, ) => { - if (!activeThreadId || !activeProject || !activeThread) return; + if (isCloudThread || !activeThreadId || !activeProject || !activeThread) return; if (options?.rememberAsLastInvoked !== false) { setLastInvokedScriptByProjectId((current) => { if (current[activeProject.id] === script.id) return current; @@ -4745,6 +4761,7 @@ export default function ChatView(props: ChatViewProps) { } }, [ + isCloudThread, activeProject, activeThread, activeThreadId, @@ -5022,18 +5039,18 @@ export default function ChatView(props: ChatViewProps) { [activeThreadRef, openPreview], ); const addDiffSurface = useCallback(() => { - if (!activeThreadRef || !isServerThread || !isGitRepo) return; + if (isCloudThread || !activeThreadRef || !isServerThread || !isGitRepo) return; useDiffPanelStore.getState().selectGitScope(activeThreadRef, "branch"); useRightPanelStore.getState().open(activeThreadRef, "diff"); onDiffPanelOpen?.(); - }, [activeThreadRef, isGitRepo, isServerThread, onDiffPanelOpen]); + }, [activeThreadRef, isCloudThread, isGitRepo, isServerThread, onDiffPanelOpen]); const openChangesFromThreadPanel = useCallback(() => { addDiffSurface(); }, [addDiffSurface]); const addFilesSurface = useCallback(() => { - if (!activeThreadRef || !activeProject) return; + if (isCloudThread || !activeThreadRef || !activeProject) return; useRightPanelStore.getState().open(activeThreadRef, "files"); - }, [activeProject, activeThreadRef]); + }, [activeProject, activeThreadRef, isCloudThread]); const supportsThreadPullRequests = serverConfig?.environment.capabilities.threadPullRequests === true; const visiblePullRequests = visibleThreadPullRequests( @@ -5132,10 +5149,10 @@ export default function ChatView(props: ChatViewProps) { }, [activePreviewMiniPlayer, activeThreadRef, deviceState.sessions, deviceStateLoaded]); const openFileSurface = useCallback( (relativePath: string) => { - if (!activeThreadRef || !activeProject) return; + if (isCloudThread || !activeThreadRef || !activeProject) return; useRightPanelStore.getState().openFile(activeThreadRef, relativePath); }, - [activeProject, activeThreadRef], + [activeProject, activeThreadRef, isCloudThread], ); // The thread's own change request, placed against the project it belongs to. Without a // project there is nothing to resolve it against, so the caller falls back to the browser. @@ -5439,7 +5456,7 @@ export default function ChatView(props: ChatViewProps) { previewPanelOpen, ]); const addTerminalSurface = useCallback(() => { - if (!activeThreadRef || !activeThreadId || !activeProject) return; + if (isCloudThread || !activeThreadRef || !activeThreadId || !activeProject) return; const cwd = gitCwd ?? activeProject.workspaceRoot; const terminalId = nextTerminalId(allocatableActiveTerminalIds); useRightPanelStore.getState().openTerminal(activeThreadRef, terminalId); @@ -5463,11 +5480,13 @@ export default function ChatView(props: ChatViewProps) { activeThreadRef, activeThreadWorktreePath, allocatableActiveTerminalIds, + isCloudThread, gitCwd, openTerminal, ]); const splitPanelTerminal = useCallback( (direction: "horizontal" | "vertical" = "horizontal") => { + if (isCloudThread) return; if ( !activeThreadRef || !activeThreadId || @@ -5498,6 +5517,7 @@ export default function ChatView(props: ChatViewProps) { }); }, [ + isCloudThread, activeProject, activeRightPanelSurface, activeThreadId, @@ -5571,13 +5591,14 @@ export default function ChatView(props: ChatViewProps) { [activeThreadRef, diffOpen, onDiffPanelOpen], ); const toggleRightPanel = useCallback(() => { + if (isCloudThread) return; if (!activeThreadRef) return; if (rightPanelOpen) { closePreviewPanel(); return; } useRightPanelStore.getState().toggleVisibility(activeThreadRef); - }, [activeThreadRef, closePreviewPanel, rightPanelOpen]); + }, [isCloudThread, activeThreadRef, closePreviewPanel, rightPanelOpen]); const toggleThreadPanel = useCallback(() => { if (!activeThreadRef) return; useRightPanelStore.getState().toggleThreadPanel(activeThreadRef, threadPanelPresentation); @@ -7430,6 +7451,22 @@ export default function ChatView(props: ChatViewProps) { return; } + if ( + isCloudThread && + [ + "terminal.toggle", + "terminal.new", + "terminal.split", + "terminal.splitVertical", + "rightPanel.toggle", + "diff.toggle", + ].includes(command ?? "") + ) { + event.preventDefault(); + event.stopPropagation(); + return; + } + if (command === "terminal.toggle") { event.preventDefault(); event.stopPropagation(); @@ -7626,6 +7663,7 @@ export default function ChatView(props: ChatViewProps) { confirmAndUnpinThread, copyActiveThreadReference, getShortcutContext, + isCloudThread, toggleRightPanel, toggleThreadPanel, toggleTerminalVisibility, @@ -8662,7 +8700,7 @@ export default function ChatView(props: ChatViewProps) { models: provider.models, modelOptions: selection.options, promptInjectionState: getComposerPromptInjectionState(messageTextForSend), - planModeEnabled: settings.planModeEnabled, + planModeEnabled: settings.planModeEnabled && !isCloudComposer, }); const text = formatOutgoingPrompt({ provider: provider.driverKind, @@ -8680,7 +8718,7 @@ export default function ChatView(props: ChatViewProps) { ), text, interactionMode: resolveComposerInteractionMode({ - planModeEnabled: settings.planModeEnabled, + planModeEnabled: settings.planModeEnabled && !isCloudComposer, provider: provider.snapshot, interactionMode: sendInteractionMode, }).interactionMode, @@ -10202,13 +10240,13 @@ export default function ChatView(props: ChatViewProps) { }, []); const onOpenTurnDiff = useCallback( (runId: RunId, filePath?: string) => { - if (!isServerThread || !activeThreadRef) return; + if (isCloudThread || !isServerThread || !activeThreadRef) return; explicitDiffOpenRef.current = diffOpen ? null : activeThreadRef; useDiffPanelStore.getState().selectTurn(activeThreadRef, runId, filePath); useRightPanelStore.getState().open(activeThreadRef, "diff"); onDiffPanelOpen?.(); }, - [activeThreadRef, diffOpen, isServerThread, onDiffPanelOpen], + [activeThreadRef, diffOpen, isCloudThread, isServerThread, onDiffPanelOpen], ); // The revert handler is read from a ref at call-time so the callback // reference is fully stable and never busts TimelineRowCtx identity. @@ -10256,162 +10294,163 @@ export default function ChatView(props: ChatViewProps) { return ; } - const rightPanelContent = activeThreadRef ? ( - renderedRightPanelSurface?.kind === "preview" ? ( - - + { + void onSend(undefined, "auto", "foreground", { annotation, image }); + }} + /> + + ) : renderedRightPanelSurface?.kind === "terminal" ? ( + { - void onSend(undefined, "auto", "foreground", { annotation, image }); - }} - /> - - ) : renderedRightPanelSurface?.kind === "terminal" ? ( - - ) : renderedRightPanelSurface?.kind === "diff" ? ( - - - - ) : renderedRightPanelSurface?.kind === "pull-request" && !pullRequestsCapabilityKnown ? ( - - ) : renderedRightPanelSurface?.kind === "pull-request" && !supportsPullRequests ? ( - - ) : renderedRightPanelSurface?.kind === "pull-request" ? ( - // No onClose: the surface tab's own X owns closing here, and a second X in the header - // would be the same action twice. The thread context also drops the checkout button, so it - // is only right for the thread's own pull request, whose branch is already under the - // reader's feet. A link the agent wrote can open any other one here, and that one has to be - // checkable out like it is anywhere else. - { - if (activeThreadRef) - useRightPanelStore.getState().openPullRequest(activeThreadRef, { - projectId: reference.projectId, - repository: reference.repository, - number: reference.number, - ...(reference.host ? { host: reference.host } : {}), - }); - }} - threadRef={activeThreadRef} - reference={{ - projectId: renderedRightPanelSurface.projectId as ProjectId, - ...(renderedRightPanelSurface.host ? { host: renderedRightPanelSurface.host } : {}), - repository: renderedRightPanelSurface.repository, - number: renderedRightPanelSurface.number, - }} - context={pullRequestPanelContext( - { - projectId: activeThread.projectId, - pullRequests: visiblePullRequests, - linkedPullRequest: linkedThreadPullRequest, - }, - renderedRightPanelSurface, - )} - composerDraftTarget={composerDraftTarget} - onBack={ - activeThreadRef !== null && pullRequestsSurfaceAvailable && visiblePullRequestCount > 1 - ? addPullRequestsSurface - : undefined - } - /> - ) : renderedRightPanelSurface?.kind === "pull-requests" && activeThreadRef ? ( - - ) : renderedRightPanelSurface?.kind === "device" ? ( - - { - closeRightPanelSurface(renderedRightPanelSurface); - useRightPanelStore.getState().show(activeThreadRef); - }} + launchContext={activeTerminalLaunchContext ?? null} + focusRequestId={terminalFocusRequestId} + keybindings={keybindings} + onAddTerminalContext={addTerminalContextToDraft} + onSplitTerminal={splitPanelTerminal} + onSplitTerminalVertical={splitPanelTerminalVertical} + onNewTerminal={addTerminalSurface} + onActiveTerminalChange={activatePanelTerminal} + onCloseTerminal={closePanelTerminal} + splitShortcutLabel={splitTerminalShortcutLabel ?? undefined} + splitVerticalShortcutLabel={splitTerminalVerticalShortcutLabel ?? undefined} + newShortcutLabel={newTerminalShortcutLabel ?? undefined} + closeShortcutLabel={closeTerminalShortcutLabel ?? undefined} + /> + ) : renderedRightPanelSurface?.kind === "diff" ? ( + + + + ) : renderedRightPanelSurface?.kind === "pull-request" && !pullRequestsCapabilityKnown ? ( + + ) : renderedRightPanelSurface?.kind === "pull-request" && !supportsPullRequests ? ( + - - ) : (renderedRightPanelSurface?.kind === "files" || - renderedRightPanelSurface?.kind === "file") && - ((activeProject && activeWorkspaceRoot) || - (renderedRightPanelSurface.kind === "file" && renderedRightPanelSurface.attachment)) ? ( - - { + if (activeThreadRef) + useRightPanelStore.getState().openPullRequest(activeThreadRef, { + projectId: reference.projectId, + repository: reference.repository, + number: reference.number, + ...(reference.host ? { host: reference.host } : {}), + }); + }} threadRef={activeThreadRef} + reference={{ + projectId: renderedRightPanelSurface.projectId as ProjectId, + ...(renderedRightPanelSurface.host ? { host: renderedRightPanelSurface.host } : {}), + repository: renderedRightPanelSurface.repository, + number: renderedRightPanelSurface.number, + }} + context={pullRequestPanelContext( + { + projectId: activeThread.projectId, + pullRequests: visiblePullRequests, + linkedPullRequest: linkedThreadPullRequest, + }, + renderedRightPanelSurface, + )} composerDraftTarget={composerDraftTarget} - keybindings={keybindings} - availableEditors={availableEditors} - relativePath={ - renderedRightPanelSurface.kind === "file" - ? renderedRightPanelSurface.relativePath - : null - } - {...(renderedRightPanelSurface.kind === "file" && renderedRightPanelSurface.attachment - ? { attachment: renderedRightPanelSurface.attachment } - : {})} - revealLine={ - renderedRightPanelSurface.kind === "file" - ? (renderedRightPanelSurface.revealLine ?? null) - : null - } - revealRequestId={ - renderedRightPanelSurface.kind === "file" - ? renderedRightPanelSurface.revealRequestId - : 0 - } - onOpenFile={openFileSurface} - onPendingChange={handleFilePendingChange} - selectedFilePending={ - renderedRightPanelSurface.kind === "file" && - pendingFileSurfaceIds.has(renderedRightPanelSurface.id) + onBack={ + activeThreadRef !== null && pullRequestsSurfaceAvailable && visiblePullRequestCount > 1 + ? addPullRequestsSurface + : undefined } - workspaceMutationId={workspaceMutationId} /> - - ) : null - ) : null; + ) : renderedRightPanelSurface?.kind === "pull-requests" && activeThreadRef ? ( + + ) : renderedRightPanelSurface?.kind === "device" ? ( + + { + closeRightPanelSurface(renderedRightPanelSurface); + useRightPanelStore.getState().show(activeThreadRef); + }} + /> + + ) : (renderedRightPanelSurface?.kind === "files" || + renderedRightPanelSurface?.kind === "file") && + ((activeProject && activeWorkspaceRoot) || + (renderedRightPanelSurface.kind === "file" && renderedRightPanelSurface.attachment)) ? ( + + + + ) : null + ) : null; const threadDetailsPanelProps: ThreadDetailsPanelProps = { anchor: threadPanelPopoverAnchorRef, handle: threadPanelPopoverHandle, @@ -10452,7 +10491,9 @@ export default function ChatView(props: ChatViewProps) { ? { onCheckoutPullRequestRequest: openPullRequestDialog } : {}), onComposerFocusRequest: scheduleComposerFocus, - ...(isServerThread && isGitRepo ? { onOpenChanges: openChangesFromThreadPanel } : {}), + ...(isServerThread && isGitRepo && !isCloudThread + ? { onOpenChanges: openChangesFromThreadPanel } + : {}), versionMismatch: showVersionMismatchBanner && versionMismatch ? { @@ -10468,7 +10509,7 @@ export default function ChatView(props: ChatViewProps) { onDeleteProjectScript: deleteProjectScript, }; const panelToggleControlProps = { - terminalAvailable: activeProject !== null, + terminalAvailable: activeProject !== null && !isCloudThread, terminalOpen: terminalUiState.terminalOpen, terminalShortcutLabel: shortcutLabelForCommand(keybindings, "terminal.toggle"), threadPanelOpen, @@ -10477,7 +10518,7 @@ export default function ChatView(props: ChatViewProps) { threadPanelShortcutLabel: shortcutLabelForCommand(keybindings, "threadPanel.toggle"), threadPanelHasAttention: activeEnvironmentUnavailableState !== null || showVersionMismatchBanner, - rightPanelAvailable: activeProject !== null, + rightPanelAvailable: activeProject !== null && !isCloudThread, rightPanelOpen, rightPanelShortcutLabel: shortcutLabelForCommand(keybindings, "rightPanel.toggle"), onToggleTerminal: toggleTerminalVisibility, @@ -10644,6 +10685,37 @@ export default function ChatView(props: ChatViewProps) { ) : null} {/* Banners overlay the timeline without changing its content height. */}
+ {isCloudThread ? ( +
+ + Kilo Cloud + {cloudExecution + ? ` · ${cloudExecution.repository} · ${cloudExecution.branch}` + : ""} + +

+ Remote repository. Local files, terminal and checkpoints are unavailable. + Closing T3 does not stop remote work or billing. +

+ {cloudExecution ? ( +

+ Last observation: {cloudExecution.observedAt ?? "unavailable"}. Task:{" "} + {cloudExecution.task}. Sandbox: {cloudExecution.sandbox}. Compute:{" "} + {cloudExecution.billing} + {cloudExecution.billingAttribution === "payer_shared" + ? " (shared account)" + : ""} + {cloudExecution.estimatedHourlyRateUsd === null + ? "" + : ` · estimated $${cloudExecution.estimatedHourlyRateUsd.toFixed(2)}/hour`} + . Inference is charged separately. +

+ ) : null} +
+ ) : null} setDismissedProviderStatusBannerKey(providerStatusBannerKey)} @@ -11169,7 +11241,7 @@ export default function ChatView(props: ChatViewProps) { key={mountedThreadKey} threadRef={mountedThreadRef} threadId={mountedThreadRef.threadId} - active={mountedThreadKey === activeThreadKey} + active={mountedThreadKey === activeThreadKey && !isCloudThread} launchContext={ mountedThreadKey === activeThreadKey ? (activeTerminalLaunchContext ?? null) : null } @@ -11217,9 +11289,9 @@ export default function ChatView(props: ChatViewProps) { onAddPullRequests={addPullRequestsSurface} onAddDevice={addDeviceSurface} browserAvailable={isPreviewSupportedInRuntime()} - terminalAvailable={activeProject !== null} - diffAvailable={isServerThread && isGitRepo} - filesAvailable={activeProject !== null} + terminalAvailable={activeProject !== null && !isCloudThread} + diffAvailable={isServerThread && isGitRepo && !isCloudThread} + filesAvailable={activeProject !== null && !isCloudThread} pullRequestAvailable={pullRequestSurfaceAvailable} pullRequestsAvailable={pullRequestsSurfaceAvailable} deviceAvailable={activeThreadRef !== null} @@ -11272,9 +11344,9 @@ export default function ChatView(props: ChatViewProps) { onAddPullRequests={addPullRequestsSurface} onAddDevice={addDeviceSurface} browserAvailable={isPreviewSupportedInRuntime()} - terminalAvailable={activeProject !== null} - diffAvailable={isServerThread && isGitRepo} - filesAvailable={activeProject !== null} + terminalAvailable={activeProject !== null && !isCloudThread} + diffAvailable={isServerThread && isGitRepo && !isCloudThread} + filesAvailable={activeProject !== null && !isCloudThread} pullRequestAvailable={pullRequestSurfaceAvailable} pullRequestsAvailable={pullRequestsSurfaceAvailable} deviceAvailable={activeThreadRef !== null} diff --git a/apps/web/src/components/chat/ProviderInstanceIcon.tsx b/apps/web/src/components/chat/ProviderInstanceIcon.tsx index e2beabd6e58b..1979c1f1346b 100644 --- a/apps/web/src/components/chat/ProviderInstanceIcon.tsx +++ b/apps/web/src/components/chat/ProviderInstanceIcon.tsx @@ -27,6 +27,7 @@ const PROVIDER_ICON_BY_PROVIDER: Partial> = { [ProviderDriverKind.make("claudeAgent")]: ClaudeAI, [ProviderDriverKind.make("opencode")]: OpenCodeIcon, [ProviderDriverKind.make("kilo")]: KiloIcon, + [ProviderDriverKind.make("kilo-cloud")]: KiloIcon, [ProviderDriverKind.make("cursor")]: CursorIcon, [ProviderDriverKind.make("grok")]: GrokIcon, [ProviderDriverKind.make("antigravity")]: AntigravityIcon, diff --git a/apps/web/src/components/settings/providerDriverMeta.ts b/apps/web/src/components/settings/providerDriverMeta.ts index 7fb52e849e7c..4a2f2ba00b8a 100644 --- a/apps/web/src/components/settings/providerDriverMeta.ts +++ b/apps/web/src/components/settings/providerDriverMeta.ts @@ -7,6 +7,7 @@ import { GrokSettings, OpenCodeSettings, KiloSettings, + KiloCloudSettings, PiSettings, ProviderDriverKind, } from "@t3tools/contracts"; @@ -48,6 +49,13 @@ export interface ProviderEnvironmentFieldDefinition { } const PROVIDER_CLIENT_DEFINITIONS: readonly ProviderClientDefinition[] = [ + { + value: ProviderDriverKind.make("kilo-cloud"), + label: "Kilo Cloud", + settingsSchema: KiloCloudSettings, + hasDefaultInstance: false, + badgeLabel: "Preview", + }, { value: ProviderDriverKind.make("kilo"), label: "Kilo", diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md index 18c97b2846d3..713b7ae623c4 100644 --- a/docs/user/providers-kilo.md +++ b/docs/user/providers-kilo.md @@ -33,5 +33,30 @@ You can select models and control tasks from web, desktop, and mobile clients connected to the environment. Configure account profiles in web or desktop settings. A disconnected client does not stop its task. -Kilo Cloud Agents are not available in this preview. No cloud task is launched and -no repository is uploaded by selecting this provider. +For remote execution, add a separate **Kilo Cloud** instance in Settings > Providers. +Select a profile signed in through the official Kilo login, a GitHub repository that +account can access, its branch, and a model. Enable paid cloud execution only when +you want prompts and that repository sent to Kilo. T3 never uploads your local +checkout or uncommitted changes. Local Kilo and Kilo Cloud can run concurrently in +separate threads; each cloud thread has its own remote worktree. + +Cloud execution currently requires **Full access**. The deployed cloud runtime +does not apply custom agent permissions, so T3 refuses restricted and Plan modes +before submitting a paid task. Shell, edits and subagents cannot be restricted in +cloud Full access. Cloud subagent history is not integrated. Use local Kilo when +you need approvals or restricted execution. Inherited Kilo profiles with setup, +MCP, skills, agents or environment variables are rejected before a new cloud task. + +Cloud prompts, native history and follow-up messages use the same remote session. +T3 reconnects by its saved task identity and does not automatically resend an +uncertain start. A cloud thread cannot use local attachments, terminals, file +checkpoints, rewind, forks or background text generation. Switching accounts does +not transfer existing tasks or stop them. + +Cloud tasks spend Kilo credit for inference and sandbox use. Automatic commits are +disabled, but an agent in Full access can still modify the remote checkout. Stop +requests inference interruption; closing a stream, task completion and sandbox +sleep are separate events. The thread shows task, sandbox and compute status +separately. Compute estimates can cover a shared account sandbox and are not a +per-task invoice. Unknown or settling status does not mean billing has stopped. +T3 does not top up credit or force a sandbox to sleep. diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index fa6c6c44ba8d..3f385c7ce4c0 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -796,6 +796,44 @@ export type OrchestrationV2PendingBackgroundTask = typeof OrchestrationV2Pending /** Provider and adapter metadata that should not overwrite the app thread's title. */ export const OrchestrationV2ProviderThreadNativeMetadata = Schema.Struct({ + cloudExecution: Schema.optional( + Schema.Struct({ + repository: Schema.String, + branch: Schema.String, + sessionId: Schema.NullOr(Schema.String), + worktreeId: Schema.NullOr(Schema.String), + task: Schema.Literals([ + "not_started", + "admission_unknown", + "queued", + "running", + "completed", + "failed", + "interrupted", + "unknown", + ]), + sandbox: Schema.Literals([ + "active", + "sleeping", + "starting", + "stopping", + "error", + "unreachable", + "unknown", + ]), + billing: Schema.Literals([ + "idle", + "active", + "stopping", + "settling", + "unavailable", + "unknown", + ]), + billingAttribution: Schema.NullOr(Schema.Literals(["payer_shared", "session"])), + estimatedHourlyRateUsd: Schema.NullOr(Schema.Number), + observedAt: Schema.NullOr(IsoDateTime), + }), + ), /** Native sessions may only be resumed within the account/configuration that created them. */ continuationKey: Schema.optional(TrimmedNonEmptyString), /** Providers that copy history with fresh message IDs preserve durable turn boundaries here. */ diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 4e5f00742477..011cf2e2f998 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1003,6 +1003,53 @@ export const KiloSettings = makeProviderSettingsSchema( ); export type KiloSettings = typeof KiloSettings.Type; +export const KiloCloudSettings = makeProviderSettingsSchema( + { + enabled: Schema.Boolean.pipe( + Schema.withDecodingDefault(Effect.succeed(false)), + Schema.annotateKey({ providerSettingsForm: { hidden: true } }), + ), + profileDirectory: TrimmedString.pipe( + Schema.withDecodingDefault(Effect.succeed("")), + Schema.annotateKey({ + title: "Kilo profile directory", + description: + "XDG root signed in with the official Kilo CLI. Personal accounts only. Credentials stay on this environment.", + }), + ), + repository: TrimmedString.pipe( + Schema.withDecodingDefault(Effect.succeed("")), + Schema.annotateKey({ + title: "Cloud repository", + description: + "GitHub owner/repository that Kilo can access. Kilo clones this repository; local files and uncommitted changes are not uploaded.", + }), + ), + branch: TrimmedNonEmptyString.pipe( + Schema.withDecodingDefault(Effect.succeed("main")), + Schema.annotateKey({ title: "Repository branch" }), + ), + model: TrimmedNonEmptyString.pipe( + Schema.withDecodingDefault(Effect.succeed("deepseek/deepseek-v4.1-flash")), + Schema.annotateKey({ + title: "Cloud model", + description: "Kilo model ID. Availability is checked by Kilo when submitting a prompt.", + }), + ), + cloudConsent: Schema.Boolean.pipe( + Schema.withDecodingDefault(Effect.succeed(false)), + Schema.annotateKey({ + title: "Allow paid cloud execution", + providerSettingsForm: { control: "switch", clearWhenEmpty: "persist" }, + description: + "Prompts and the selected repository are sent to Kilo. Model and sandbox charges use your Kilo account. Closing T3 does not stop a remote task or guarantee billing has stopped. T3 never tops up credit. Cloud execution currently requires Full access, including remote shell, edits and subagents. Automatic commits are disabled.", + }), + ), + }, + { order: ["profileDirectory", "repository", "branch", "model", "cloudConsent"] }, +); +export type KiloCloudSettings = typeof KiloCloudSettings.Type; + /** * A read-only quota source outside this environment's provider CLIs. The * only kind today is a CLIProxyAPI hub, whose management API reports the From 77049894900f22ec95742f01a883b422e20441c0 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 16:12:36 +0000 Subject: [PATCH 08/44] fix(web): keep cloud composer separate from local checkout controls --- apps/desktop/scripts/kilo-ui-evidence.mjs | 1 + .../Adapters/KiloCloudAdapterV2.test.ts | 77 ++++++++++++++++++- apps/web/src/components/ChatView.tsx | 8 +- 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/apps/desktop/scripts/kilo-ui-evidence.mjs b/apps/desktop/scripts/kilo-ui-evidence.mjs index 17fd657a2451..75495ac636d2 100644 --- a/apps/desktop/scripts/kilo-ui-evidence.mjs +++ b/apps/desktop/scripts/kilo-ui-evidence.mjs @@ -237,6 +237,7 @@ try { await page.getByRole("button", { name: "Providers", exact: true }).click(); await page.waitForURL("**/settings/providers*"); await page.getByRole("button", { name: "Add provider", exact: true }).waitFor(); + await page.getByRole("button", { name: "Select Kilo Cloud", exact: true }).click(); if (process.env.KILO_CLOUD_TEST_PROFILE) { const consent = page.getByRole("switch", { name: "Allow paid cloud execution", exact: true }); await consent.click(); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index d986ffce105d..4b20302f9700 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -48,6 +48,9 @@ const fixture = Effect.acquireRelease( interruptAccepted: false, interruptPosts: 0, permission: false, + question: false, + questionPosts: 0, + questionAnswers: null as unknown, answerAccepted: false, answerPosts: 0, }; @@ -100,6 +103,12 @@ const fixture = Effect.acquireRelease( if (control.interruptAccepted) control.status = "interrupted"; return reply({ success: control.interruptAccepted }); } + if (operation === "cloudAgentNext.answerQuestion") { + control.questionPosts++; + control.questionAnswers = input.answers; + control.question = false; + return reply({ success: true }); + } if (operation === "cloudAgentNext.answerPermission") { control.answerPosts++; if (control.answerAccepted) control.permission = false; @@ -144,7 +153,26 @@ const fixture = Effect.acquireRelease( }, ] : [], - questions: [], + questions: control.question + ? [ + { + id: "question-fixture", + sessionID: state.native, + questions: [ + { + header: "Files", + question: "Which files?", + multiple: true, + custom: false, + options: [ + { label: "README.md", description: "Documentation" }, + { label: "fixture.py", description: "Synthetic code" }, + ], + }, + ], + }, + ] + : [], }); if (operation === "cloudAgentNext.getMessageResult") return reply({ @@ -431,24 +459,39 @@ it.live( yield* Fiber.interrupt(restoredEvents); remote.control.status = "running"; remote.control.permission = true; + remote.control.question = true; remote.control.incompleteHistory = true; const pendingRequest = yield* Deferred.make< Extract >(); + const pendingQuestion = + yield* Deferred.make< + Extract + >(); + const questionResolved = yield* Deferred.make(); const resolvedItems: Array = []; const failedWithoutHistory = yield* Deferred.make(); yield* restored.events.pipe( Stream.runForEach((event) => Effect.gen(function* () { resolvedItems.push(event); + if ( + event.type === "turn_item.updated" && + event.turnItem.type === "user_input_request" && + event.turnItem.status === "completed" + ) + yield* Deferred.succeed(questionResolved, undefined); if (event.type === "turn.terminal" && event.status === "failed") yield* Deferred.succeed(failedWithoutHistory, undefined); if ( event.type === "runtime_request.updated" && event.runtimeRequest.status === "pending" ) - yield* Deferred.succeed(pendingRequest, event); + yield* Deferred.succeed( + event.runtimeRequest.kind === "user_input" ? pendingQuestion : pendingRequest, + event, + ); }), ), Effect.forkScoped, @@ -467,6 +510,36 @@ it.live( text: "Follow up in the same workspace", }, }); + const question = yield* Deferred.await(pendingQuestion); + const unanswered = yield* restored + .respondToRuntimeRequest({ requestId: question.runtimeRequest.id, answers: {} }) + .pipe(Effect.flip); + assert.include(unanswered.message, "requires an answer"); + assert.equal(remote.control.questionPosts, 0); + yield* restored.respondToRuntimeRequest({ + requestId: question.runtimeRequest.id, + answers: { "0": ["README.md", "fixture.py"] }, + }); + yield* Deferred.await(questionResolved); + assert.equal(remote.control.questionPosts, 1); + assert.deepEqual(remote.control.questionAnswers, [["README.md", "fixture.py"]]); + assert.isTrue( + resolvedItems.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "user_input_request" && + event.turnItem.questions[0]?.multiSelect === true && + event.turnItem.questions[0]?.allowCustomAnswer === false, + ), + ); + assert.isTrue( + resolvedItems.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "user_input_request" && + event.turnItem.status === "completed", + ), + ); const pending = yield* Deferred.await(pendingRequest); yield* restored .respondToRuntimeRequest({ requestId: pending.runtimeRequest.id, decision: "accept" }) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 21d3155825bb..daf1e5f2ab1b 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -4101,16 +4101,16 @@ export default function ChatView(props: ChatViewProps) { const mountComposerContextStrip = shouldShowComposerContextStrip({ isDraftHeroState, persistInActiveThreads: settings.persistComposerContextStrip, - hasActiveProject: activeProject !== null && !showProviderSubagentBar, - isGitRepo, + hasActiveProject: activeProject !== null && !showProviderSubagentBar && !isCloudComposer, + isGitRepo: isGitRepo && !isCloudComposer, showEnvironmentIndicator: showComposerEnvironmentIndicator, hostsRestingComposerControls: routeKind === "server", }); const showComposerContextStrip = shouldShowComposerContextStrip({ isDraftHeroState, persistInActiveThreads: settings.persistComposerContextStrip, - hasActiveProject: activeProject !== null && !showProviderSubagentBar, - isGitRepo, + hasActiveProject: activeProject !== null && !showProviderSubagentBar && !isCloudComposer, + isGitRepo: isGitRepo && !isCloudComposer, showEnvironmentIndicator: showComposerEnvironmentIndicator, hostsRestingComposerControls: routeKind === "server" && restingComposerControlsVisible, }); From 3bedf8ced67c3bd65c2580854d940d90996fcf0d Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 17:02:37 +0000 Subject: [PATCH 09/44] chore: remove temporary Kilo contribution workflow --- .github/workflows/kilo-provider.yml | 148 ---------------------------- 1 file changed, 148 deletions(-) delete mode 100644 .github/workflows/kilo-provider.yml diff --git a/.github/workflows/kilo-provider.yml b/.github/workflows/kilo-provider.yml deleted file mode 100644 index 309b3fce72c0..000000000000 --- a/.github/workflows/kilo-provider.yml +++ /dev/null @@ -1,148 +0,0 @@ -name: Kilo provider checks - -on: - pull_request: - paths: - - .github/workflows/kilo-provider.yml - - apps/server/src/provider/kilo/** - - apps/server/src/provider/Drivers/Kilo*Driver.ts - - apps/server/src/provider/builtInDrivers.ts - - apps/server/src/orchestration-v2/** - - apps/server/src/textGeneration/KiloTextGeneration.ts - - apps/server/scripts/kilo-stream-benchmark.mjs - - apps/desktop/scripts/kilo-ui-evidence.mjs - - apps/web/src/components/** - - apps/mobile/src/** - - packages/contracts/src/** - - packages/client-runtime/** - - third-party-licenses.config.json - - knip.jsonc - - apps/server/package.json - - pnpm-lock.yaml - -permissions: - contents: read - -concurrency: - group: kilo-provider-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - provider-integration: - # The upstream suite uses Blacksmith runners unavailable to this fork. - # Reassess this workflow before proposing the provider upstream. - # The owner's former login redirects; the repository ID survives that rename. - if: github.repository_id == '1286185343' - runs-on: ubuntu-24.04 - timeout-minutes: 25 - steps: - - uses: actions/checkout@v6 - with: - # Verify the exact PR head rather than an implicit merge commit. - ref: ${{ github.event.pull_request.head.sha }} - - - uses: voidzero-dev/setup-vp@v1 - with: - node-version-file: package.json - cache: true - run-install: true - - - name: Install the pinned local Kilo CLI - run: npm install --prefix "$RUNNER_TEMP/kilo-probe" --ignore-scripts @kilocode/cli@7.8.3 - - - name: Lint changed provider and client code - run: >- - vp lint apps/server/src/provider/kilo - apps/server/src/provider/Drivers/KiloDriver.ts - apps/server/src/provider/Drivers/KiloCloudDriver.ts - apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts - apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts - apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.live.test.ts - apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts - apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts - apps/server/src/textGeneration/KiloTextGeneration.ts - apps/server/scripts/kilo-stream-benchmark.mjs - apps/desktop/scripts/kilo-ui-evidence.mjs - apps/web/src/components/Icons.tsx - apps/web/src/components/chat/ProviderInstanceIcon.tsx - apps/web/src/components/settings/providerDriverMeta.ts - packages/client-runtime/src/kiloIcon.ts - packages/contracts/src/settings.ts - packages/contracts/src/orchestrationV2.ts --deny-warnings - - - name: Lint mobile changes - run: vp lint apps/mobile/src/components/ProviderIcon.tsx apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx - - - name: Check changed workspace types - run: | - vp exec tsc --noEmit -p apps/server - vp exec tsc --noEmit -p apps/web - vp exec tsc --noEmit -p apps/mobile - vp exec tsc --noEmit -p packages/contracts - vp exec tsc --noEmit -p packages/client-runtime - - - name: Check server dependencies and exports - run: vp exec knip --workspace apps/server --workspace apps/web --workspace packages/contracts --workspace packages/client-runtime --include files,dependencies,exports --preprocessor ./scripts/knip-schemas.ts --no-config-hints - - - name: Check mobile files and dependencies - run: vp exec knip --workspace apps/mobile --include files,dependencies --no-config-hints - - - name: Test native Kilo sessions and OpenCode regressions - env: - KILO_BIN: ${{ runner.temp }}/kilo-probe/node_modules/.bin/kilo - KILO_TEST_ROOT: ${{ runner.temp }} - run: >- - vp test run --no-file-parallelism - apps/server/src/provider/kilo - apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts - apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts - apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts - apps/server/src/orchestration-v2/ProviderSessionManager.test.ts - apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts - apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts - apps/server/src/orchestration-v2/RunExecutionService.test.ts - apps/server/src/orchestration-v2/Adapters/OpenCodeAdapterV2.test.ts - apps/server/src/provider/opencodeRuntime.environment.test.ts - apps/server/src/provider/opencodeRuntime.permissions.test.ts - apps/server/src/provider/Drivers/OpenCodeDriver.test.ts - - - name: Build the server bundle - run: vp run --filter t3 build:bundle - - - name: Build web and desktop bundles - run: | - sudo apt-get update - sudo apt-get install -y pkg-config libsecret-1-dev - # The desktop build includes its web and server dependencies. - vp run --filter @t3tools/desktop build - - - name: Export the Android JavaScript bundle - working-directory: apps/mobile - env: - APP_VARIANT: development - run: vp exec expo export --platform android --output-dir "$RUNNER_TEMP/kilo-android" - - - name: Install Chromium for the authorized UI check - run: node apps/desktop/node_modules/playwright-core/cli.js install --with-deps chromium - - - name: Exercise the real UI with local inference - env: - KILO_BIN: ${{ runner.temp }}/kilo-probe/node_modules/.bin/kilo - KILO_EVIDENCE_DIR: ${{ runner.temp }}/kilo-evidence - run: node apps/desktop/scripts/kilo-ui-evidence.mjs - - - name: Compare equivalent stream workloads against the unchanged SDK layer - run: | - mkdir -p "$RUNNER_TEMP/kilo-baseline" "$RUNNER_TEMP/kilo-evidence" - git fetch --no-tags --depth=1 origin dfa68127e1a4a992b4184babe0e193432465bdd2 - git show dfa68127e1a4a992b4184babe0e193432465bdd2:apps/server/src/provider/kilo/KiloSessionClient.ts > "$RUNNER_TEMP/kilo-baseline/KiloSessionClient.ts" - ln -s "$GITHUB_WORKSPACE/apps/server/node_modules" "$RUNNER_TEMP/kilo-baseline/node_modules" - node --expose-gc apps/server/scripts/kilo-stream-benchmark.mjs "$RUNNER_TEMP/kilo-baseline/KiloSessionClient.ts" > "$RUNNER_TEMP/kilo-evidence/stream-benchmark.json" - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: kilo-ui-${{ github.event.pull_request.head.sha }} - path: ${{ runner.temp }}/kilo-evidence - if-no-files-found: warn - retention-days: 14 From 3edb86e51a24b7d9c92e5d20136ed52ddeec107d Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 17:11:58 +0000 Subject: [PATCH 10/44] fix(kilo): retire local sessions when profile credentials change --- .../SettingsProviderAccountsRouteScreen.tsx | 5 +- .../server/src/provider/Drivers/KiloDriver.ts | 24 ++++- .../provider/kilo/KiloRuntime.live.test.ts | 102 ++++++++++++++++++ apps/server/src/provider/kilo/KiloRuntime.ts | 78 ++++++++++++++ .../src/provider/kilo/KiloSessionClient.ts | 13 ++- docs/user/providers-kilo.md | 6 ++ 6 files changed, 222 insertions(+), 6 deletions(-) diff --git a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx index dc4fdd7bd55c..07d1cd3c6677 100644 --- a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx @@ -56,8 +56,11 @@ export function SettingsProviderAccountsRouteScreen() { {provider.displayName} + {provider.message ? ( + {provider.message} + ) : null} - {(provider.message ?? provider.driver === "kilo-cloud") + {provider.driver === "kilo-cloud" ? "Kilo Cloud runs in a remote repository and uses Kilo credit. Closing T3 does not stop remote work or billing. Manage its account and repository in web or desktop Settings." : "Kilo runs on this environment. Manage its isolated account profile in web or desktop Settings."} diff --git a/apps/server/src/provider/Drivers/KiloDriver.ts b/apps/server/src/provider/Drivers/KiloDriver.ts index 1772910f50e4..52682b53d235 100644 --- a/apps/server/src/provider/Drivers/KiloDriver.ts +++ b/apps/server/src/provider/Drivers/KiloDriver.ts @@ -81,9 +81,21 @@ export const KiloDriver: ProviderDriver = { input.instanceId, encodeURIComponent(input.config.accountId), ); + const authContent = yield* KiloRuntime.readAuth(profileDirectory, environment).pipe( + Effect.mapError( + () => + new ProviderDriverError({ + driver: kind, + instanceId: input.instanceId, + detail: + "Could not read the selected Kilo credentials. Check the profile and reload the provider.", + }), + ), + ); const identity = [ path.resolve(profileDirectory), input.config.accountId, + authContent, ...input.environment .toSorted((a, b) => a.name.localeCompare(b.name)) .map((entry) => `${entry.name}=${entry.value}`), @@ -105,7 +117,17 @@ export const KiloDriver: ProviderDriver = { binaryPath: input.config.binaryPath, profileDirectory, environment, - }); + authContent, + }).pipe( + Effect.mapError( + () => + new ProviderDriverError({ + driver: kind, + instanceId: input.instanceId, + detail: "Could not prepare the selected Kilo credentials.", + }), + ), + ); const orchestrationAdapter = yield* KiloAdapter.make({ instanceId: input.instanceId, continuationKey, diff --git a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts index b059ba85c328..98eaa9d431f4 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts @@ -1,13 +1,18 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; +import { ProviderInstanceId, ProviderSessionId, ThreadId } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Scope from "effect/Scope"; import { describe } from "vite-plus/test"; import * as KiloRuntime from "./KiloRuntime.ts"; +import { KiloDriver } from "../Drivers/KiloDriver.ts"; +import * as ServerConfig from "../../config.ts"; +import * as IdAllocator from "../../orchestration-v2/IdAllocator.ts"; const binary = process.env.KILO_BIN; const environment = { @@ -23,6 +28,103 @@ const environment = { }; describe.runIf(binary !== undefined)("KiloRuntime native lifecycle", () => { + it.live( + "retires live clients and rejects saved threads after credentials change in the same profile", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-account-change-" }); + const authDir = path.join(root, "data", "kilo"); + const authFile = path.join(authDir, "auth.json"); + yield* fs.makeDirectory(authDir, { recursive: true }); + const credentials = (key: string) => JSON.stringify({ kilo: { type: "api", key } }); + yield* fs.writeFileString(authFile, credentials("synthetic-account-a")); + const runtime = yield* KiloRuntime.make({ + instanceId: "account-test", + binaryPath: binary!, + profileDirectory: root, + environment, + }); + const connection = yield* runtime.open(root); + const native = yield* connection.client.create([]); + const create = KiloDriver.create({ + instanceId: ProviderInstanceId.make("account-test"), + displayName: undefined, + enabled: false, + config: { ...KiloDriver.defaultConfig(), binaryPath: binary!, profileDirectory: root }, + environment: Object.entries(environment).flatMap(([name, value]) => + value === undefined ? [] : [{ name, value, sensitive: false }], + ), + }); + const verify = Effect.gen(function* () { + const first = yield* create; + const request = { + threadId: ThreadId.make("account-test"), + providerSessionId: ProviderSessionId.make("account-test"), + modelSelection: { instanceId: first.instanceId, model: "fixture/test" }, + runtimePolicy: { + runtimeMode: "full-access" as const, + interactionMode: "default" as const, + cwd: root, + }, + }; + const firstSession = yield* first.orchestrationAdapter.openSession(request); + const thread = yield* firstSession.ensureThread(request); + const unchanged = yield* create; + assert.deepStrictEqual(unchanged.continuationIdentity, first.continuationIdentity); + const resumed = yield* unchanged.orchestrationAdapter.openSession(request); + assert.equal( + (yield* resumed.resumeThread({ providerThread: thread })).nativeThreadRef?.nativeId, + thread.nativeThreadRef?.nativeId, + ); + yield* fs.writeFileString(authFile, credentials("synthetic-account-b")); + const failure = yield* connection.client.read(native).pipe(Effect.flip); + assert.equal(failure.reason, "wrong_owner"); + yield* connection.exitCode.pipe(Effect.timeout("5 seconds")); + assert.isFalse(yield* connection.isRunning); + yield* runtime.open(root).pipe(Effect.flip); + const replacement = yield* create; + assert.notEqual( + replacement.continuationIdentity.continuationKey, + first.continuationIdentity.continuationKey, + ); + const secondSession = yield* replacement.orchestrationAdapter.openSession(request); + const rejected = yield* secondSession + .resumeThread({ providerThread: thread }) + .pipe(Effect.flip); + assert.include(rejected.message, "account or configuration changed"); + const fresh = yield* secondSession.ensureThread(request); + assert.notEqual(fresh.nativeThreadRef?.nativeId, thread.nativeThreadRef?.nativeId); + }); + yield* verify.pipe( + Effect.provide( + Layer.mergeAll( + ServerConfig.layerTest(root, { prefix: "t3-kilo-driver-" }), + IdAllocator.layer, + ), + ), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + { timeout: 60000 }, + ); + + it.live("rejects malformed credentials without exposing them or falling back to disk", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-auth-" }); + for (const content of ["", "secret-not-json", "null", "[]"]) { + const failure = yield* KiloRuntime.readAuth(root, { KILO_AUTH_CONTENT: content }).pipe( + Effect.flip, + ); + assert.equal(failure.operation, "authentication"); + assert.notInclude(failure.message, "secret-not-json"); + assert.isUndefined(failure.cause); + } + assert.equal(yield* KiloRuntime.readAuth(root, {}), "{}"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.live( "isolates profiles, closes owned processes and resumes after restart", () => diff --git a/apps/server/src/provider/kilo/KiloRuntime.ts b/apps/server/src/provider/kilo/KiloRuntime.ts index 4ba712c18f60..1d28d9fc7eda 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.ts @@ -43,6 +43,44 @@ export class KiloRuntime extends Context.Service< } >()("t3/provider/kilo/KiloRuntime") {} +const authSchema = Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)); +const decodeAuth = Schema.decodeUnknownEffect(authSchema); +const encodeAuth = Schema.encodeEffect(authSchema); + +/** Freeze the selected credential source; never fall back after a read/parse failure. */ +export const readAuth = Effect.fn("KiloRuntime.readAuth")(function* ( + profileDirectory: string, + environment: NodeJS.ProcessEnv, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return yield* Effect.gen(function* () { + const contents = + environment.KILO_AUTH_CONTENT ?? + (yield* fs + .readFileString(path.join(profileDirectory, "data", "kilo", "auth.json")) + .pipe( + Effect.catchTag("PlatformError", (error) => + error.reason._tag === "NotFound" ? Effect.succeed("{}") : Effect.fail(error), + ), + )); + const auth = yield* decodeAuth(contents); + // Stable key order keeps harmless formatting changes from retiring sessions. + return yield* encodeAuth( + Object.fromEntries(Object.entries(auth).toSorted(([a], [b]) => a.localeCompare(b))), + ); + }).pipe( + Effect.mapError( + () => + new KiloRuntimeError({ + operation: "authentication", + detail: + "Could not read the selected Kilo credentials. Check the profile and reload the provider.", + }), + ), + ); +}); + /** Every open owns a process. Registry replacement closes the old account's process scopes. */ export const make = Effect.fn("KiloRuntime.make")(function* (input: { readonly instanceId: string; @@ -50,6 +88,7 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { /** An XDG root for this account, not the Kilo data directory itself. */ readonly profileDirectory: string; readonly environment: NodeJS.ProcessEnv; + readonly authContent?: string; }) { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fs = yield* FileSystem.FileSystem; @@ -60,6 +99,7 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { const fail = (operation: string, detail: string) => (cause: unknown) => new KiloRuntimeError({ operation, detail, cause }); const profile = path.resolve(input.profileDirectory); + const authContent = input.authContent ?? (yield* readAuth(profile, input.environment)); const environment: NodeJS.ProcessEnv = { ...input.environment, XDG_CONFIG_HOME: path.join(profile, "config"), @@ -70,8 +110,29 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { // Background children outlive root turns and need a separate T3 continuation contract. KILO_EXPERIMENTAL_BACKGROUND_SUBAGENTS: "false", KILO_SERVER_USERNAME: "kilo", + // The CLI supports this immutable credential source. A login in the selected + // profile cannot change credentials underneath an already running process. + KILO_AUTH_CONTENT: authContent, }; let closed = false; + const checkAuth = Effect.gen(function* () { + const current = yield* readAuth(profile, input.environment); + if (closed || current !== authContent) { + closed = true; + return yield* new KiloRuntimeError({ + operation: "authentication", + detail: "Kilo credentials changed. Reload the provider and start a new thread.", + }); + } + }).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.tapError(() => + Effect.sync(() => { + closed = true; + }), + ), + ); yield* Scope.addFinalizer( owner, Effect.sync(() => { @@ -86,6 +147,7 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { operation: "open", detail: "Kilo account runtime has been retired.", }); + yield* checkAuth; const caller = yield* Effect.scope; const scope = yield* Scope.fork(owner); yield* Scope.addFinalizer(caller, Scope.close(scope, Exit.void)); @@ -146,6 +208,13 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { ), ); yield* Effect.addFinalizer(() => cleanup); + const guard = checkAuth.pipe(Effect.onError(() => cleanup)); + // Observe idle or in-flight account replacement as well as request boundaries. + // Never read credential files once per SSE event. + yield* Effect.forever(Effect.sleep("250 millis").pipe(Effect.andThen(guard))).pipe( + Effect.ignore, + Effect.forkIn(scope), + ); const ready = yield* Deferred.make(); let output = ""; yield* child.stdout.pipe( @@ -186,6 +255,15 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { directory, baseUrl: url, serverPassword: password, + beforeRequest: guard.pipe( + Effect.mapError( + () => + new KiloSessionClient.KiloSessionError({ + operation: "authentication", + reason: "wrong_owner", + }), + ), + ), }).pipe( Effect.mapError( fail( diff --git a/apps/server/src/provider/kilo/KiloSessionClient.ts b/apps/server/src/provider/kilo/KiloSessionClient.ts index 496c802625e2..404bb78b99f1 100644 --- a/apps/server/src/provider/kilo/KiloSessionClient.ts +++ b/apps/server/src/provider/kilo/KiloSessionClient.ts @@ -108,6 +108,7 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { readonly baseUrl: string; readonly serverPassword?: string; readonly serverUsername?: string; + readonly beforeRequest?: Effect.Effect; }) { const client = createKiloClient({ baseUrl: input.baseUrl, @@ -125,10 +126,13 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { }); const request = (operation: string, run: (signal: AbortSignal) => Promise<{ data?: A }>) => - Effect.tryPromise({ - try: run, - catch: (cause) => new KiloSessionError({ operation, reason: "request_failed", cause }), - }).pipe( + (input.beforeRequest ?? Effect.void).pipe( + Effect.andThen( + Effect.tryPromise({ + try: run, + catch: (cause) => new KiloSessionError({ operation, reason: "request_failed", cause }), + }), + ), Effect.timeout("10 seconds"), Effect.catchTag( "TimeoutError", @@ -139,6 +143,7 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { ? Effect.fail(new KiloSessionError({ operation, reason: "invalid_response" })) : Effect.succeed(response.data), ), + Effect.tap(() => input.beforeRequest ?? Effect.void), ); const acknowledge = (operation: string) => (accepted: unknown) => diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md index 713b7ae623c4..c5e4030912c1 100644 --- a/docs/user/providers-kilo.md +++ b/docs/user/providers-kilo.md @@ -16,6 +16,12 @@ or instance environment retires its running processes. Existing threads cannot resume under the replacement account. A Ready status confirms local CLI readiness; it does not prove that a model account is authenticated or has available credit. +Local processes use a fixed credential snapshot. Replacing credentials in the same +profile retires its processes and prevents old threads from resuming after reload. +Reload the provider after login or token refresh and start a new thread. Credential +refresh is conservatively treated as an account change because local account +identity cannot be verified without contacting each model provider. + Prompts run in the selected T3 workspace. Use separate T3 worktrees for tasks that must not share files. Separate conversation IDs alone do not isolate a checkout. Stop terminates the task's owned local process group. A later prompt can restore From ee9f6654539484425c5fe9272458f6de4b5aa4ea Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 17:44:58 +0000 Subject: [PATCH 11/44] fix(kilo): block external plugins before native session startup --- .../provider/kilo/KiloRuntime.live.test.ts | 43 +++++++++++++++++++ apps/server/src/provider/kilo/KiloRuntime.ts | 4 ++ docs/user/providers-kilo.md | 12 ++++++ 3 files changed, 59 insertions(+) diff --git a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts index 98eaa9d431f4..68ed14fb0d81 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts @@ -7,6 +7,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; import * as Scope from "effect/Scope"; +import * as Schema from "effect/Schema"; import { describe } from "vite-plus/test"; import * as KiloRuntime from "./KiloRuntime.ts"; @@ -15,6 +16,7 @@ import * as ServerConfig from "../../config.ts"; import * as IdAllocator from "../../orchestration-v2/IdAllocator.ts"; const binary = process.env.KILO_BIN; +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const environment = { PATH: process.env.PATH, HTTP_PROXY: process.env.HTTP_PROXY, @@ -28,6 +30,47 @@ const environment = { }; describe.runIf(binary !== undefined)("KiloRuntime native lifecycle", () => { + it.live( + "does not execute repository or external plugins before session permissions", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-plugin-" }); + const cwd = path.join(root, "checkout"); + const pluginDir = path.join(cwd, ".kilo", "plugins"); + yield* fs.makeDirectory(pluginDir, { recursive: true }); + const marker = path.join(root, "repository-plugin-ran"); + const explicitMarker = path.join(root, "explicit-plugin-ran"); + const body = (target: string) => + `import { writeFileSync } from "node:fs";\nwriteFileSync(${encodeJson(target)}, "executed");\nexport const fixture = async () => ({});\n`; + yield* fs.writeFileString(path.join(pluginDir, "unsafe.ts"), body(marker)); + const explicitPlugin = path.join(root, "explicit.ts"); + yield* fs.writeFileString(explicitPlugin, body(explicitMarker)); + // A profile override must not reopen the approval bypass. + const runtime = yield* KiloRuntime.make({ + instanceId: "plugins", + binaryPath: binary!, + profileDirectory: path.join(root, "profile"), + environment: { + ...environment, + KILO_DISABLE_PROJECT_CONFIG: "0", + KILO_PURE: "0", + KILO_CONFIG_CONTENT: encodeJson({ plugin: [explicitPlugin] }), + }, + }); + const connection = yield* runtime.open(cwd); + yield* connection.client.models(); + const ref = yield* connection.client.create([ + { permission: "*", pattern: "*", action: "ask" }, + ]); + assert.equal((yield* connection.client.read(ref)).id, ref.sessionId); + assert.isFalse(yield* fs.exists(marker)); + assert.isFalse(yield* fs.exists(explicitMarker)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + { timeout: 30000 }, + ); + it.live( "retires live clients and rejects saved threads after credentials change in the same profile", () => diff --git a/apps/server/src/provider/kilo/KiloRuntime.ts b/apps/server/src/provider/kilo/KiloRuntime.ts index 1d28d9fc7eda..19bb626fd257 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.ts @@ -107,6 +107,10 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { XDG_CACHE_HOME: path.join(profile, "cache"), XDG_STATE_HOME: path.join(profile, "state"), KILO_DISABLE_AUTOUPDATE: "1", + // Repository config and external plugins execute before session permissions. + // Keep these forced after instance overrides, including in Full access. + KILO_DISABLE_PROJECT_CONFIG: "1", + KILO_PURE: "1", // Background children outlive root turns and need a separate T3 continuation contract. KILO_EXPERIMENTAL_BACKGROUND_SUBAGENTS: "false", KILO_SERVER_USERNAME: "kilo", diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md index c5e4030912c1..175f74994c59 100644 --- a/docs/user/providers-kilo.md +++ b/docs/user/providers-kilo.md @@ -22,6 +22,18 @@ Reload the provider after login or token refresh and start a new thread. Credent refresh is conservatively treated as an account change because local account identity cannot be verified without contacting each model provider. +Repository Kilo configuration and external plugins are disabled. Plugins execute +outside session approval rules, so instance environment settings cannot enable +them. Configure models in the selected account profile or explicit instance +configuration instead. Kilo's built-in authentication plugins remain available. + +**Known security limitation:** CLI 7.8.3 still imports legacy `.kilo/mcp.json` and +`.kilocode/mcp.json` despite disabling project configuration. Their MCP commands +can start before session permission checks. The CLI has no supported blanket MCP +disable, and configuration reloads make a preflight check insufficient. Restricted +execution is therefore not safe for untrusted repositories with these settings. +Disabling external plugins does not resolve this separate limitation. + Prompts run in the selected T3 workspace. Use separate T3 worktrees for tasks that must not share files. Separate conversation IDs alone do not isolate a checkout. Stop terminates the task's owned local process group. A later prompt can restore From 16a4ff02eb0977e27d05e65130664463b5b60eef Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 18:39:58 +0000 Subject: [PATCH 12/44] fix(kilo): refuse unsafe local runtime and isolate cloud actions --- apps/desktop/scripts/kilo-ui-evidence.mjs | 80 +++--- .../features/files/ThreadFilesRouteScreen.tsx | 9 +- .../review/useReviewHeaderPresentation.tsx | 1 + .../src/features/review/useReviewSections.ts | 3 +- .../terminal/ThreadTerminalRouteScreen.tsx | 15 + .../features/threads/ThreadGitControls.tsx | 40 ++- .../features/threads/ThreadRouteScreen.tsx | 20 +- .../src/state/threadLocalWorkspace.test.ts | 56 ++++ apps/mobile/src/state/threadLocalWorkspace.ts | 24 ++ .../state/use-selected-thread-git-actions.ts | 8 +- .../state/use-selected-thread-git-state.ts | 10 +- .../src/state/use-selected-thread-worktree.ts | 22 +- .../Adapters/KiloAdapterV2.live.test.ts | 22 +- .../Adapters/KiloAdapterV2.ts | 41 ++- .../Adapters/KiloCloudAdapterV2.test.ts | 43 ++- .../Adapters/KiloCloudAdapterV2.ts | 71 ++++- .../RunExecutionService.test.ts | 228 +++++++++------ .../orchestration-v2/RunExecutionService.ts | 16 +- .../src/provider/Drivers/KiloCloudDriver.ts | 13 +- .../server/src/provider/Drivers/KiloDriver.ts | 5 +- .../provider/kilo/KiloCloudAccount.test.ts | 62 ++++ .../src/provider/kilo/KiloCloudAccount.ts | 33 ++- .../src/provider/kilo/KiloCloudClient.test.ts | 23 +- .../src/provider/kilo/KiloCloudClient.ts | 3 +- .../provider/kilo/KiloCloudWebClient.test.ts | 147 ++++++++++ .../src/provider/kilo/KiloCloudWebClient.ts | 114 ++++++-- .../provider/kilo/KiloRuntime.live.test.ts | 3 +- .../provider/kilo/KiloRuntime.safety.test.ts | 99 +++++++ apps/server/src/provider/kilo/KiloRuntime.ts | 12 + .../src/textGeneration/KiloTextGeneration.ts | 22 +- apps/web/src/components/ChatMarkdown.test.tsx | 68 ++++- apps/web/src/components/ChatMarkdown.tsx | 39 ++- apps/web/src/components/ChatView.tsx | 272 ++++++++++-------- docs/user/providers-kilo.md | 90 +++--- 34 files changed, 1307 insertions(+), 407 deletions(-) create mode 100644 apps/mobile/src/state/threadLocalWorkspace.test.ts create mode 100644 apps/mobile/src/state/threadLocalWorkspace.ts create mode 100644 apps/server/src/provider/kilo/KiloCloudAccount.test.ts create mode 100644 apps/server/src/provider/kilo/KiloRuntime.safety.test.ts diff --git a/apps/desktop/scripts/kilo-ui-evidence.mjs b/apps/desktop/scripts/kilo-ui-evidence.mjs index 75495ac636d2..9e5882d7cd70 100644 --- a/apps/desktop/scripts/kilo-ui-evidence.mjs +++ b/apps/desktop/scripts/kilo-ui-evidence.mjs @@ -10,6 +10,9 @@ import * as NodeUtil from "node:util"; import { chromium } from "playwright-core"; if (!process.env.KILO_BIN) throw new Error("KILO_BIN must point to the pinned local CLI"); +const verifyBlocked = process.env.KILO_VERIFY_BLOCKED === "1"; +if (verifyBlocked && process.env.KILO_CLOUD_TEST_PROFILE) + throw new Error("Blocked-runtime verification must use no live cloud profile"); const root = NodePath.resolve(import.meta.dirname, "../../.."); const temporary = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-kilo-ui-")); const evidence = process.env.KILO_EVIDENCE_DIR ?? NodePath.join(temporary, "evidence"); @@ -195,48 +198,58 @@ try { await page.getByText("Local folder", { exact: true }).click(); await page.getByPlaceholder("Enter path (e.g. ~/projects/my-app)").fill(workspace); await page.getByPlaceholder("Enter path (e.g. ~/projects/my-app)").press("Enter"); - if (process.env.KILO_CLOUD_TEST_PROFILE) { + if (!verifyBlocked) { + if (process.env.KILO_CLOUD_TEST_PROFILE) { + await page.locator("[data-chat-provider-model-picker-label]").click(); + await page.getByPlaceholder("Search models...").fill("deepseek-v4.1-flash"); + await page.getByText("deepseek/deepseek-v4.1-flash", { exact: true }).last().click(); + await page + .getByText("Closing T3 does not stop remote work or billing.", { exact: false }) + .waitFor(); + await page.getByRole("button", { name: "Unknown", exact: true }).click(); + await page.getByRole("menuitemradio", { name: /^Low/ }).click(); + await page.getByRole("button", { name: "Low", exact: true }).waitFor(); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "cloud-before-send.png"), + }); + } await page.locator("[data-chat-provider-model-picker-label]").click(); - await page.getByPlaceholder("Search models...").fill("deepseek-v4.1-flash"); - await page.getByText("deepseek/deepseek-v4.1-flash", { exact: true }).last().click(); - await page - .getByText("Closing T3 does not stop remote work or billing.", { exact: false }) - .waitFor(); - await page.getByRole("button", { name: "Unknown", exact: true }).click(); - await page.getByRole("menuitemradio", { name: /^Low/ }).click(); - await page.getByRole("button", { name: "Low", exact: true }).waitFor(); + await page.getByPlaceholder("Search models...").fill("Local fixture"); + await page.getByText("Local fixture", { exact: true }).last().click(); + await page.getByRole("button", { name: "Local fixture", exact: true }).waitFor(); + await page.locator("[contenteditable=true]").fill("Kilo local integration: say hello."); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "before-send.png"), + }); + await page.getByRole("button", { name: "Submit message", exact: true }).click(); + await page.getByText(answer, { exact: true }).waitFor({ timeout: 60000 }); await page.screenshot({ animations: "disabled", - path: NodePath.join(evidence, "cloud-before-send.png"), + path: NodePath.join(evidence, "streamed-answer.png"), }); + await page.getByRole("button", { name: "Submit message", exact: true }).waitFor(); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "completed-answer.png"), + }); + console.log("Local native answer rendered; opening provider settings."); } - await page.locator("[data-chat-provider-model-picker-label]").click(); - await page.getByPlaceholder("Search models...").fill("Local fixture"); - await page.getByText("Local fixture", { exact: true }).last().click(); - await page.getByRole("button", { name: "Local fixture", exact: true }).waitFor(); - await page.locator("[contenteditable=true]").fill("Kilo local integration: say hello."); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "before-send.png"), - }); - await page.getByRole("button", { name: "Submit message", exact: true }).click(); - await page.getByText(answer, { exact: true }).waitFor({ timeout: 60000 }); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "streamed-answer.png"), - }); - await page.getByRole("button", { name: "Submit message", exact: true }).waitFor(); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "completed-answer.png"), - }); - console.log("Local native answer rendered; opening provider settings."); await page.getByRole("button", { name: "Settings", exact: true }).click(); await page.waitForURL("**/settings/general*"); await page.getByText("Restore device defaults", { exact: true }).waitFor(); await page.getByRole("button", { name: "Providers", exact: true }).click(); await page.waitForURL("**/settings/providers*"); await page.getByRole("button", { name: "Add provider", exact: true }).waitFor(); + if (verifyBlocked) { + await page.getByRole("button", { name: "Select Kilo", exact: true }).click(); + await page.getByText("Local Kilo execution is disabled:", { exact: false }).waitFor(); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "local-execution-blocked.png"), + }); + } await page.getByRole("button", { name: "Select Kilo Cloud", exact: true }).click(); if (process.env.KILO_CLOUD_TEST_PROFILE) { const consent = page.getByRole("switch", { name: "Allow paid cloud execution", exact: true }); @@ -260,13 +273,16 @@ try { path: NodePath.join(evidence, "provider-settings.png"), }); await context.close(); - if (!requests) throw new Error("The real CLI did not contact the local inference fixture"); + if (verifyBlocked && requests !== 0) throw new Error("Blocked execution reached inference"); + if (!verifyBlocked && !requests) + throw new Error("The real CLI did not contact the local inference fixture"); await NodeFSP.writeFile( NodePath.join(evidence, "verification.json"), JSON.stringify( { commit: (await execFile("git", ["rev-parse", "HEAD"], { cwd: root })).stdout.trim(), inferenceRequests: requests, + localExecutionBlocked: verifyBlocked, inference: "loopback fixture only", client: "Chromium web", }, diff --git a/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx b/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx index 06b6664431b5..f0ffd5de22e3 100644 --- a/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx +++ b/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx @@ -351,7 +351,7 @@ function useThreadFilesWorkspace(params: { const routeCwd = firstRouteParam(params.cwd); const routeProjectName = firstRouteParam(params.projectName); const { selectedThread, selectedThreadProject } = useThreadSelection(); - const { selectedThreadCwd } = useSelectedThreadWorktree(); + const { selectedThreadCwd, localWorkspaceEnabled } = useSelectedThreadWorktree(); const environmentId = routeEnvironmentId !== null ? EnvironmentId.make(routeEnvironmentId) @@ -363,7 +363,12 @@ function useThreadFilesWorkspace(params: { } | null; return { - cwd: routeCwd ?? selectedThreadCwd ?? project?.workspaceRoot ?? null, + cwd: + routeThreadId !== null + ? localWorkspaceEnabled + ? selectedThreadCwd + : null + : (routeCwd ?? project?.workspaceRoot ?? null), environmentId, projectName: routeProjectName ?? project?.title ?? "Files", selectedThread, diff --git a/apps/mobile/src/features/review/useReviewHeaderPresentation.tsx b/apps/mobile/src/features/review/useReviewHeaderPresentation.tsx index f9ab9d794a47..9f1bedd80dbf 100644 --- a/apps/mobile/src/features/review/useReviewHeaderPresentation.tsx +++ b/apps/mobile/src/features/review/useReviewHeaderPresentation.tsx @@ -43,6 +43,7 @@ export function useReviewHeaderPresentation(props: { const gitMenuAvailable = selectedThread !== null && String(selectedThread.id) === String(props.threadId); const gitMenu = useThreadGitMenuDefinition({ + gitControlsEnabled: props.selectedThreadCwd !== null, environmentId: props.environmentId, threadId: props.threadId, currentBranch: selectedThread?.branch ?? null, diff --git a/apps/mobile/src/features/review/useReviewSections.ts b/apps/mobile/src/features/review/useReviewSections.ts index ede2d808c8b0..0ade91ab5c75 100644 --- a/apps/mobile/src/features/review/useReviewSections.ts +++ b/apps/mobile/src/features/review/useReviewSections.ts @@ -34,7 +34,8 @@ export function useReviewSections(input: { readonly reviewCache: ReviewCacheForThread; }) { const { environmentId, reviewCache, threadId } = input; - const enabled = input.enabled ?? true; + const { localWorkspaceEnabled } = useSelectedThreadWorktree(); + const enabled = localWorkspaceEnabled && (input.enabled ?? true); const selectedThread = useSelectedThreadProjection(); const { selectedThreadCwd } = useSelectedThreadWorktree(); const diffPreview = useEnvironmentQuery( diff --git a/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx b/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx index c1a0d9ee65c1..2e0cfa275269 100644 --- a/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx +++ b/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx @@ -1,3 +1,4 @@ +import { useSelectedThreadWorktree } from "../../state/use-selected-thread-worktree"; import { DEFAULT_TERMINAL_ID, EnvironmentId, ThreadId } from "@t3tools/contracts"; import { type KnownTerminalSession } from "@t3tools/client-runtime/state/terminal"; import { SymbolView } from "../../components/AppSymbol"; @@ -240,6 +241,20 @@ type ThreadTerminalRouteScreenProps = StaticScreenProps<{ }>; export function ThreadTerminalRouteScreen(props: ThreadTerminalRouteScreenProps) { + const { localWorkspaceEnabled } = useSelectedThreadWorktree(); + if (!localWorkspaceEnabled) + return ( + + + + ); + return ; +} + +function LocalThreadTerminalRouteScreen(props: ThreadTerminalRouteScreenProps) { const insets = useSafeAreaInsets(); const navigation = useNavigation(); const writeTerminal = useAtomCommand(terminalEnvironment.write, "terminal write"); diff --git a/apps/mobile/src/features/threads/ThreadGitControls.tsx b/apps/mobile/src/features/threads/ThreadGitControls.tsx index 18471f88af09..df9b21ca707d 100644 --- a/apps/mobile/src/features/threads/ThreadGitControls.tsx +++ b/apps/mobile/src/features/threads/ThreadGitControls.tsx @@ -81,6 +81,7 @@ type QuickActionIcon = /** The subset of git-control wiring the standalone git menu needs. */ export type ThreadGitMenuProps = { + readonly gitControlsEnabled?: boolean; readonly environmentId: EnvironmentId | string; readonly threadId: ThreadId | string; readonly currentBranch: string | null; @@ -112,13 +113,14 @@ type ThreadGitControlsProps = ThreadGitMenuProps & { function useThreadGitControlModel(props: ThreadGitMenuProps) { const navigation = useNavigation(); + const enabled = props.gitControlsEnabled !== false; const environmentId = props.environmentId; const threadId = props.threadId; const { gitStatus, gitOperationLabel, onPull, onRunAction } = props; const currentBranchLabel = gitStatus?.refName ?? props.currentBranch ?? "Detached HEAD"; const busy = gitOperationLabel !== null; - const isRepo = gitStatus?.isRepo ?? true; + const isRepo = enabled && (gitStatus?.isRepo ?? true); const hasPrimaryRemote = gitStatus?.hasPrimaryRemote ?? false; const isDefaultRef = gitStatus?.isDefaultRef ?? false; @@ -163,6 +165,7 @@ function useThreadGitControlModel(props: ThreadGitMenuProps) { const runActionWithPrompt = useCallback( async (input: GitActionRequestInput) => { + if (!enabled) return; const confirmableAction = input.action === "push" || input.action === "create_pr" || @@ -191,10 +194,11 @@ function useThreadGitControlModel(props: ThreadGitMenuProps) { await onRunAction(input); }, - [environmentId, gitStatus, isDefaultRef, onRunAction, navigation, threadId], + [enabled, environmentId, gitStatus, isDefaultRef, onRunAction, navigation, threadId], ); const runQuickAction = useCallback(async () => { + if (!enabled) return; if (quickAction.kind === "open_pr") { await openExistingPr(); return; @@ -206,9 +210,10 @@ function useThreadGitControlModel(props: ThreadGitMenuProps) { if (quickAction.kind === "run_action" && quickAction.action) { await runActionWithPrompt({ action: quickAction.action }); } - }, [onPull, openExistingPr, quickAction, runActionWithPrompt]); + }, [enabled, onPull, openExistingPr, quickAction, runActionWithPrompt]); const openFiles = useCallback(() => { + if (!enabled) return; if (props.onOpenFilesInspector) { props.onOpenFilesInspector(); return; @@ -217,16 +222,18 @@ function useThreadGitControlModel(props: ThreadGitMenuProps) { environmentId: String(environmentId), threadId: String(threadId), }); - }, [environmentId, props.onOpenFilesInspector, navigation, threadId]); + }, [enabled, environmentId, props.onOpenFilesInspector, navigation, threadId]); const openReview = useCallback(() => { + if (!enabled) return; navigation.navigate("ThreadReview", { environmentId: EnvironmentId.make(String(environmentId)), threadId: ThreadId.make(String(threadId)), }); - }, [environmentId, navigation, threadId]); + }, [enabled, environmentId, navigation, threadId]); const openGitInspector = useCallback(() => { + if (!enabled) return; if (props.onOpenGitInspector) { props.onOpenGitInspector(); return; @@ -235,7 +242,7 @@ function useThreadGitControlModel(props: ThreadGitMenuProps) { environmentId: String(environmentId), threadId: String(threadId), }); - }, [environmentId, props.onOpenGitInspector, navigation, threadId]); + }, [enabled, environmentId, props.onOpenGitInspector, navigation, threadId]); return { currentBranchLabel, @@ -324,6 +331,7 @@ function useThreadGitHeaderActionItems(props: ThreadGitControlsProps): ThreadGit }, git: { accessibilityLabel: "Git actions", + disabled: props.gitControlsEnabled === false, icon: { name: "point.topleft.down.curvedto.point.bottomright.up", type: "sfSymbol" }, identifier: "thread-right-git", label: "Git", @@ -396,6 +404,7 @@ function useThreadGitHeaderActionItems(props: ThreadGitControlsProps): ThreadGit props.canOpenFiles, props.canOpenTerminal, props.gitStatus, + props.gitControlsEnabled, props.onMergeBack, props.onOpenNewTerminal, props.onOpenTerminal, @@ -409,16 +418,22 @@ function useThreadGitHeaderActionItems(props: ThreadGitControlsProps): ThreadGit export function useThreadGitRightHeaderItems(props: ThreadGitControlsProps): HeaderItems { const actionItems = useThreadGitHeaderActionItems(props); return useMemo( - () => [actionItems.git, actionItems.files, actionItems.terminal] as HeaderItems, - [actionItems], + () => + props.gitControlsEnabled === false + ? [] + : ([actionItems.git, actionItems.files, actionItems.terminal] as HeaderItems), + [actionItems, props.gitControlsEnabled], ); } export function useThreadGitCenterHeaderItems(props: ThreadGitControlsProps): HeaderItems { const actionItems = useThreadGitHeaderActionItems(props); return useMemo( - () => [actionItems.files, actionItems.git, actionItems.terminal] as HeaderItems, - [actionItems], + () => + props.gitControlsEnabled === false + ? [] + : ([actionItems.files, actionItems.git, actionItems.terminal] as HeaderItems), + [actionItems, props.gitControlsEnabled], ); } @@ -426,7 +441,7 @@ export function ThreadGitControls(props: ThreadGitControlsProps) { const model = useThreadGitControlModel(props); const showActionControls = props.showActionControls ?? true; - if (!showActionControls) { + if (!showActionControls || props.gitControlsEnabled === false) { return null; } @@ -523,7 +538,8 @@ export function ThreadGitMenu(props: ThreadGitMenuProps) { /** Returns menu data because native toolbars serialize direct items rather than rendering component children. */ export function useThreadGitMenuDefinition(props: ThreadGitMenuProps): ScreenHeaderMenu | null { - return threadGitMenuDefinition(props, useThreadGitControlModel(props)); + const model = useThreadGitControlModel(props); + return props.gitControlsEnabled === false ? null : threadGitMenuDefinition(props, model); } function threadGitMenuDefinition( diff --git a/apps/mobile/src/features/threads/ThreadRouteScreen.tsx b/apps/mobile/src/features/threads/ThreadRouteScreen.tsx index bc562354812d..14cfda6f8dc2 100644 --- a/apps/mobile/src/features/threads/ThreadRouteScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadRouteScreen.tsx @@ -336,7 +336,7 @@ function ThreadRouteContent( } = useThreadSelection(); const selectedThreadDetailState = props.selectedThreadDetailState; const selectedThreadDetail = Option.getOrNull(selectedThreadDetailState.data); - const { selectedThreadCwd } = useSelectedThreadWorktree(); + const { selectedThreadCwd, localWorkspaceEnabled } = useSelectedThreadWorktree(); const composer = useThreadComposerState(); const gitState = useSelectedThreadGitState(); const gitActions = useSelectedThreadGitActions(); @@ -374,6 +374,7 @@ function ThreadRouteContent( const mergeBackBusyRef = useRef(false); const handleMergeBack = useCallback(async () => { if ( + !localWorkspaceEnabled || mergeBackBusyRef.current || !selectedThread || mergeBackTargetThreadId === null || @@ -400,7 +401,14 @@ function ThreadRouteContent( } finally { mergeBackBusyRef.current = false; } - }, [mergeBack, mergeBackRun, mergeBackTargetThreadId, navigation, selectedThread]); + }, [ + localWorkspaceEnabled, + mergeBack, + mergeBackRun, + mergeBackTargetThreadId, + navigation, + selectedThread, + ]); const params = props.route.params; const environmentIdRaw = firstRouteParam(params.environmentId); const environmentId = environmentIdRaw ? EnvironmentId.make(environmentIdRaw) : null; @@ -825,10 +833,14 @@ function ThreadRouteContent( ], ); const threadGitControlProps = { + gitControlsEnabled: localWorkspaceEnabled, environmentId: environmentIdRaw ?? "", threadId: threadId ?? "", auxiliaryPaneControl: - !layout.usesSplitView && fileInspector.supported && selectedThreadCwd !== null + localWorkspaceEnabled && + !layout.usesSplitView && + fileInspector.supported && + selectedThreadCwd !== null ? { accessibilityLabel: "Toggle inspector", onPress: handleToggleInspector, @@ -841,7 +853,7 @@ function ThreadRouteContent( onOpenGitInspector: !isCloudThread && fileInspector.supported ? handleOpenGitInspector : undefined, onMergeBack: - mergeBackTargetThreadId !== null && mergeBackRun !== null + localWorkspaceEnabled && mergeBackTargetThreadId !== null && mergeBackRun !== null ? () => void handleMergeBack() : undefined, currentBranch: selectedThread?.branch ?? null, diff --git a/apps/mobile/src/state/threadLocalWorkspace.test.ts b/apps/mobile/src/state/threadLocalWorkspace.test.ts new file mode 100644 index 000000000000..71decf636cb9 --- /dev/null +++ b/apps/mobile/src/state/threadLocalWorkspace.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vite-plus/test"; +import { + ProviderDriverKind, + ProviderThreadId, + type OrchestrationV2ThreadProjection, +} from "@t3tools/contracts"; +import { threadLocalWorkspace } from "./threadLocalWorkspace"; + +describe("mobile thread workspace routing", () => { + const local = { + driver: ProviderDriverKind.make("codex"), + providerThreads: [], + activeProviderThreadId: null, + worktreePath: "/local/worktree", + workspaceRoot: "/local/repository", + }; + it("keeps local worktree and repository actions available for a known local provider", () => { + expect(threadLocalWorkspace(local)).toEqual({ + localWorkspaceEnabled: true, + selectedThreadWorktreePath: "/local/worktree", + selectedThreadCwd: "/local/worktree", + selectedThreadGitRootCwd: "/local/repository", + }); + expect(threadLocalWorkspace({ ...local, worktreePath: null }).selectedThreadCwd).toBe( + "/local/repository", + ); + }); + it("never supplies a local Git/file target for cloud, missing providers or persisted cloud history", () => { + const persisted = [ + { id: "remote", nativeMetadata: { cloudExecution: { sessionId: "workspace_remote" } } }, + ] as unknown as OrchestrationV2ThreadProjection["providerThreads"]; + for (const input of [ + { ...local, driver: ProviderDriverKind.make("kilo-cloud") }, + { ...local, driver: undefined }, + { ...local, activeProviderThreadId: ProviderThreadId.make("not-loaded") }, + { + ...local, + activeProviderThreadId: ProviderThreadId.make("cloud-before-admission"), + providerThreads: [ + { id: "cloud-before-admission", driver: "kilo-cloud" }, + ] as unknown as OrchestrationV2ThreadProjection["providerThreads"], + }, + { + ...local, + providerThreads: persisted, + activeProviderThreadId: ProviderThreadId.make("remote"), + }, + ]) + expect(threadLocalWorkspace(input)).toEqual({ + localWorkspaceEnabled: false, + selectedThreadWorktreePath: null, + selectedThreadCwd: null, + selectedThreadGitRootCwd: null, + }); + }); +}); diff --git a/apps/mobile/src/state/threadLocalWorkspace.ts b/apps/mobile/src/state/threadLocalWorkspace.ts new file mode 100644 index 000000000000..ea5c3105aae2 --- /dev/null +++ b/apps/mobile/src/state/threadLocalWorkspace.ts @@ -0,0 +1,24 @@ +import type { OrchestrationV2ThreadProjection, ProviderDriverKind } from "@t3tools/contracts"; + +/** Never substitute a local checkout for a remote or unresolved conversation. */ +export function threadLocalWorkspace(input: { + readonly driver: ProviderDriverKind | undefined; + readonly providerThreads: OrchestrationV2ThreadProjection["providerThreads"]; + readonly activeProviderThreadId: OrchestrationV2ThreadProjection["thread"]["activeProviderThreadId"]; + readonly worktreePath: string | null; + readonly workspaceRoot: string | null; +}) { + const active = input.providerThreads.find((thread) => thread.id === input.activeProviderThreadId); + const cloud = + input.driver === "kilo-cloud" || + active?.driver === "kilo-cloud" || + !!active?.nativeMetadata?.cloudExecution; + const resolved = input.activeProviderThreadId === null || active !== undefined; + const enabled = resolved && !cloud && input.driver !== undefined; + return { + localWorkspaceEnabled: enabled, + selectedThreadWorktreePath: enabled ? input.worktreePath : null, + selectedThreadCwd: enabled ? (input.worktreePath ?? input.workspaceRoot) : null, + selectedThreadGitRootCwd: enabled ? input.workspaceRoot : null, + }; +} diff --git a/apps/mobile/src/state/use-selected-thread-git-actions.ts b/apps/mobile/src/state/use-selected-thread-git-actions.ts index e66f690428e8..a0ee160c9add 100644 --- a/apps/mobile/src/state/use-selected-thread-git-actions.ts +++ b/apps/mobile/src/state/use-selected-thread-git-actions.ts @@ -36,7 +36,8 @@ export function useSelectedThreadGitActions() { const createWorktree = useAtomCommand(vcsEnvironment.createWorktree, { reportFailure: false }); const pull = useAtomCommand(vcsEnvironment.pull, { reportFailure: false }); const { selectedThread, selectedThreadProject } = useThreadSelection(); - const { selectedThreadCwd, selectedThreadWorktreePath } = useSelectedThreadWorktree(); + const { selectedThreadCwd, selectedThreadWorktreePath, selectedThreadGitRootCwd } = + useSelectedThreadWorktree(); const runStackedAction = useAtomCommand( vcsActionManager.runStackedAction({ environmentId: selectedThread?.environmentId ?? null, @@ -45,7 +46,6 @@ export function useSelectedThreadGitActions() { { reportFailure: false }, ); - const selectedThreadGitRootCwd = selectedThreadProject?.workspaceRoot ?? null; const branchTarget = useMemo( () => ({ environmentId: selectedThread?.environmentId ?? null, @@ -77,7 +77,7 @@ export function useSelectedThreadGitActions() { const refreshSelectedThreadGitStatus = useCallback( async (options?: { readonly quiet?: boolean; readonly cwd?: string | null }) => { - if (!selectedThread || !selectedThreadProject) { + if (!selectedThread || !selectedThreadProject || !selectedThreadCwd) { return null; } @@ -116,7 +116,7 @@ export function useSelectedThreadGitActions() { ); useEffect(() => { - if (!selectedThread || !selectedThreadProject) { + if (!selectedThread || !selectedThreadProject || !selectedThreadCwd) { return; } void refreshSelectedThreadGitStatus({ quiet: true }); diff --git a/apps/mobile/src/state/use-selected-thread-git-state.ts b/apps/mobile/src/state/use-selected-thread-git-state.ts index a8c037db6f77..75eabf4f8fa5 100644 --- a/apps/mobile/src/state/use-selected-thread-git-state.ts +++ b/apps/mobile/src/state/use-selected-thread-git-state.ts @@ -10,8 +10,8 @@ import { useThreadSelection } from "./use-thread-selection"; import { useSelectedThreadWorktree } from "./use-selected-thread-worktree"; export function useSelectedThreadGitState() { - const { selectedThread, selectedThreadProject } = useThreadSelection(); - const { selectedThreadCwd } = useSelectedThreadWorktree(); + const { selectedThread } = useThreadSelection(); + const { selectedThreadCwd, selectedThreadGitRootCwd } = useSelectedThreadWorktree(); const selectedThreadGitTarget = useMemo( () => ({ @@ -22,7 +22,7 @@ export function useSelectedThreadGitState() { ); const gitActionState = useVcsActionState(selectedThreadGitTarget); const sourceControlDiscovery = useEnvironmentQuery( - selectedThread === null + selectedThread === null || selectedThreadCwd === null ? null : sourceControlEnvironment.discovery({ environmentId: selectedThread.environmentId, @@ -33,10 +33,10 @@ export function useSelectedThreadGitState() { const selectedThreadBranchTarget = useMemo( () => ({ environmentId: selectedThread?.environmentId ?? null, - cwd: selectedThreadProject?.workspaceRoot ?? null, + cwd: selectedThreadGitRootCwd, query: null, }), - [selectedThread?.environmentId, selectedThreadProject?.workspaceRoot], + [selectedThread?.environmentId, selectedThreadGitRootCwd], ); const selectedThreadBranchState = useBranches(selectedThreadBranchTarget); const selectedThreadBranches = useMemo( diff --git a/apps/mobile/src/state/use-selected-thread-worktree.ts b/apps/mobile/src/state/use-selected-thread-worktree.ts index 8b1727831f62..30663376eaca 100644 --- a/apps/mobile/src/state/use-selected-thread-worktree.ts +++ b/apps/mobile/src/state/use-selected-thread-worktree.ts @@ -1,11 +1,16 @@ +import * as Option from "effect/Option"; import { useMemo } from "react"; -import { useSelectedThreadWorktreePath } from "./use-thread-detail"; +import { useSelectedThreadWorktreePath, useSelectedThreadDetailState } from "./use-thread-detail"; import { useThreadSelection } from "./use-thread-selection"; import { resolvePreferredThreadWorktreePath } from "../features/terminal/terminalLaunchContext"; +import { threadLocalWorkspace } from "./threadLocalWorkspace"; + export function useSelectedThreadWorktree() { - const { selectedThread, selectedThreadProject } = useThreadSelection(); + const { selectedThread, selectedThreadProject, selectedEnvironmentRuntime } = + useThreadSelection(); + const projection = Option.getOrNull(useSelectedThreadDetailState().data); const detailWorktreePath = useSelectedThreadWorktreePath(); const selectedThreadWorktreePath = useMemo( @@ -17,8 +22,13 @@ export function useSelectedThreadWorktree() { [detailWorktreePath, selectedThread?.worktreePath], ); - return { - selectedThreadWorktreePath, - selectedThreadCwd: selectedThreadWorktreePath ?? selectedThreadProject?.workspaceRoot ?? null, - }; + return threadLocalWorkspace({ + driver: selectedEnvironmentRuntime?.serverConfig?.providers.find( + (provider) => provider.instanceId === selectedThread?.providerInstanceId, + )?.driver, + providerThreads: projection?.providerThreads ?? [], + activeProviderThreadId: selectedThread?.activeProviderThreadId ?? null, + worktreePath: selectedThreadWorktreePath, + workspaceRoot: selectedThreadProject?.workspaceRoot ?? null, + }); } diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts index 9f3225e5961c..e8d9e82cd800 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts @@ -57,7 +57,19 @@ const inference = Effect.acquireRelease( body += String(chunk); }); req.on("end", () => { - const parsed = JSON.parse(body) as Record; + let parsed: Record; + try { + parsed = JSON.parse(body) as Record; + } catch { + res.writeHead(400); + res.end(); + return; + } + if (!parsed || !Array.isArray(parsed.messages)) { + res.writeHead(400); + res.end(); + return; + } requests.push(parsed); if (parsed.stream !== true) { res.writeHead(200, { "Content-Type": "application/json" }); @@ -84,7 +96,7 @@ const inference = Effect.acquireRelease( if ( (control.mode === "subagent" || control.mode === "subagent-approval") && messages.at(-1)?.role !== "tool" && - !JSON.stringify(messages.at(-1)).includes("Child fixture reply") + !JSON.stringify(messages.at(-1) ?? null).includes("Child fixture reply") ) { res.write( `data: ${JSON.stringify({ @@ -232,7 +244,8 @@ const inference = Effect.acquireRelease( }), ); -describe.runIf(binary !== undefined)("Kilo adapter with native runtime and local inference", () => { +// Historical native conformance coverage. Re-enable only after an audited MCP runtime fix. +describe.skip("Kilo adapter with native runtime and local inference", () => { it.live( "delivers a real streamed turn and restores its native history", () => @@ -631,7 +644,8 @@ describe.runIf(binary !== undefined)("Kilo adapter with native runtime and local text: "Delegate the local child fixture", }, }); - yield* Deferred.await(terminal); + const ended = yield* Deferred.await(terminal); + assert.equal(ended.type === "turn.terminal" ? ended.status : undefined, "completed"); assert.isFalse(seen.slice(before).some((event) => event.type === "app_thread.created")); } model.control.mode = "json"; diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts index 3fcce44db8a7..f5fb9a95fce5 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts @@ -18,6 +18,7 @@ import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; +import * as Schema from "effect/Schema"; import * as Queue from "effect/Queue"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; @@ -38,6 +39,10 @@ import { openCodePermissionRules } from "./OpenCodeAdapterV2.ts"; import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { openCodeToolTurnItem } from "./OpenCodeToolItems.ts"; +const isKiloRuntimeError = Schema.is(KiloRuntime.KiloRuntimeError); + +const isKiloSessionError = Schema.is(KiloSessionError); + export const KILO_PROVIDER = ProviderDriverKind.make("kilo"); const capabilities: OrchestrationV2ProviderCapabilities = { @@ -140,7 +145,20 @@ const nativeRef = (nativeId: string) => ({ strength: "strong" as const, }); const wire = (effect: Effect.Effect) => - effect.pipe(Effect.mapError(() => error("Kilo request failed; the operation was not retried"))); + effect.pipe( + Effect.mapError( + (cause) => + new Adapter.ProviderAdapterProtocolError({ + driver: KILO_PROVIDER, + // These typed messages contain no raw transport responses or credentials. + detail: + isKiloRuntimeError(cause) || isKiloSessionError(cause) + ? cause.message + : "Kilo request failed; the operation was not retried", + cause, + }), + ), + ); // Kilo 7.8.3 task.ts persists inherited edit/bash/MCP ceilings before launching a child. const permissions = (policy: Adapter.ProviderAdapterV2RuntimePolicy) => { @@ -1006,7 +1024,15 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { if (previous) messages.set(entry.info.id, previous); roles.set(entry.info.id, entry.info.role); if (entry.info.id === active?.messageID) active.admitted = true; - for (const part of entry.parts) putPart(part); + for (const part of entry.parts) { + putPart(part); + if ( + part.type === "tool" && + part.tool === "task" && + parents.get(part.messageID) === active?.messageID + ) + yield* task(part); + } // Snapshot reads must not re-publish historical records over T3's durable run metadata. yield* messageFromParts( entry.info.id, @@ -1189,7 +1215,16 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { eventFiber = yield* watch.pipe(Effect.forkIn(scope)); yield* Deferred.await(ready).pipe( Effect.timeout("10 seconds"), - Effect.mapError(() => error("Kilo stream readiness timed out")), + Effect.catchTag("TimeoutError", () => + Effect.fail(error("Kilo stream readiness timed out")), + ), + Effect.onError(() => + Effect.gen(function* () { + if (eventFiber) yield* Fiber.interrupt(eventFiber); + eventFiber = undefined; + subscribed = false; + }), + ), ); subscribed = true; }); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index 4b20302f9700..de16417cd367 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -43,6 +43,8 @@ const fixture = Effect.acquireRelease( }); const control = { status: "completed", + dropNextPrepare: false, + omittedItemCount: 0, missingHistory: false, incompleteHistory: false, interruptAccepted: false, @@ -92,8 +94,21 @@ const fixture = Effect.acquireRelease( messages: [{ id: input.initialMessageId!, prompt: input.prompt! }], }; conversations.set(state.cloud, state); + if (control.dropNextPrepare) { + control.dropNextPrepare = false; + response.destroy(); + return; + } return reply({ cloudAgentSessionId: state.cloud, kiloSessionId: state.native }); } + if (operation === "cliSessionsV2.list") + return reply({ + cliSessions: [...conversations.values()].map((state) => ({ + session_id: state.native, + cloud_agent_session_id: state.cloud, + })), + nextCursor: null, + }); const state = [...conversations.values()].find( (item) => item.cloud === input.cloudAgentSessionId || item.native === input.session_id, ); @@ -174,6 +189,20 @@ const fixture = Effect.acquireRelease( ] : [], }); + if (operation === "cloudAgentNext.getSandboxStatus") + return reply({ + status: control.status === "running" ? "active" : "sleeping", + observedAt: 1, + inactivityTimeoutMs: null, + estimatedSleepAt: null, + }); + if (operation === "cloudAgentNext.getComputeBillingStatus") + return reply({ + phase: control.status === "running" ? "active" : "idle", + attribution: "session", + estimatedHourlyRateMicrodollars: 0, + estimatedIntervalAmountMicrodollars: 0, + }); if (operation === "cloudAgentNext.getMessageResult") return reply({ cloudAgentSessionId: state.cloud, @@ -188,7 +217,7 @@ const fixture = Effect.acquireRelease( watermarkEventId: 3, history: { nextCursor: null, - omittedItemCount: 0, + omittedItemCount: control.omittedItemCount, messages: state.messages.flatMap((message) => [ { info: { @@ -260,6 +289,7 @@ it.live( () => Effect.gen(function* () { const remote = yield* fixture; + remote.control.dropNextPrepare = true; const fs = yield* FileSystem.FileSystem; const directory = yield* fs.makeTempDirectoryScoped(); const journal = yield* Journal.make(directory); @@ -472,10 +502,17 @@ it.live( const questionResolved = yield* Deferred.make(); const resolvedItems: Array = []; const failedWithoutHistory = yield* Deferred.make(); + const failedAndSleeping = yield* Deferred.make(); yield* restored.events.pipe( Stream.runForEach((event) => Effect.gen(function* () { resolvedItems.push(event); + if ( + event.type === "provider_thread.updated" && + event.providerThread.nativeMetadata?.cloudExecution?.task === "failed" && + event.providerThread.nativeMetadata.cloudExecution.sandbox === "sleeping" + ) + yield* Deferred.succeed(failedAndSleeping, undefined); if ( event.type === "turn_item.updated" && event.turnItem.type === "user_input_request" && @@ -598,7 +635,7 @@ it.live( assert.include(denied.message, "Paid cloud execution is disabled"); assert.equal(remote.submissions(), 2); remote.control.status = "failed"; - remote.control.missingHistory = true; + remote.control.omittedItemCount = 1; yield* restored.startTurn({ ...restore, reattach: false, @@ -614,6 +651,8 @@ it.live( }, }); yield* Deferred.await(failedWithoutHistory); + yield* Deferred.await(failedAndSleeping); + assert.isFalse(yield* restored.hasPendingBackgroundWork!); assert.equal((yield* journal.read).at(-1)?.state, "failed"); assert.equal(remote.submissions(), 2); const first = (yield* journal.read)[0]!; diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts index 03f0540c0198..36d8c23c5e3c 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -10,6 +10,9 @@ import { type ProviderInstanceId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Crypto from "effect/Crypto"; import * as Deferred from "effect/Deferred"; import * as DateTime from "effect/DateTime"; @@ -196,6 +199,9 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { let binding: Cloud.CloudBinding | undefined; let watching = false; let streamWatching = false; + let streamFiber: Fiber.Fiber | undefined; + let admissionProbeAt = 0; + let admissionProbeDelay = 2_000; let streamCursor = 0; let monitorSandbox = false; let taskState: @@ -363,6 +369,9 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { providerSession: { ...session, status: "ready", updatedAt: at, lastError: null }, }); active = undefined; + // Terminal task state must not pin a history retry forever. Reopening + // the thread can retry history independently of the ended turn. + needsHistoryRestore = false; if (!binding) monitorSandbox = false; taskState = terminal; yield* Deferred.succeed(terminalSignal, undefined); @@ -607,6 +616,10 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { const expectedMessageId = active?.messageId; if (!binding) { if (active && !active.prepared) { + const now = yield* Clock.currentTimeMillis; + if (now < admissionProbeAt) return; + admissionProbeAt = now + admissionProbeDelay; + admissionProbeDelay = Math.min(admissionProbeDelay * 2, 60_000); const found = yield* wire( options.client.findAdmission(options.repository, active.messageId), ); @@ -731,6 +744,8 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ? null : billing.estimatedHourlyRateMicrodollars / 1_000_000, }; + if (!active && sandbox?.status === "sleeping" && billing?.phase === "idle") + monitorSandbox = false; const signature = encode(snapshot); if (signature !== lifecycleSignature) { lifecycleSignature = signature; @@ -743,14 +758,12 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { }; yield* emit({ type: "provider_thread.updated", driver, providerThread: thread }); } - if (!active && sandbox?.status === "sleeping" && billing?.phase === "idle") - monitorSandbox = false; }); const watchEvents = Effect.gen(function* () { if (streamWatching || !binding) return; streamWatching = true; const ownedBinding = binding; - yield* Effect.gen(function* () { + streamFiber = yield* Effect.gen(function* () { // State is changed by the reconciler while this reader observes notifications. // oxlint-disable-next-line no-unmodified-loop-condition while (active || needsHistoryRestore || monitorSandbox) { @@ -779,10 +792,16 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ); if (active || needsHistoryRestore || monitorSandbox) yield* Effect.sleep("5 seconds"); } - streamWatching = false; - }).pipe(Effect.forkIn(scope)); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + streamWatching = false; + }), + ), + Effect.forkIn(scope), + ); }); - const watch = Effect.gen(function* () { + const watch: Effect.Effect = Effect.gen(function* () { if (watching) { yield* Queue.offer(wake, undefined); return; @@ -793,6 +812,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { // Reconcile and lifecycle update this session state. // oxlint-disable-next-line no-unmodified-loop-condition while (active || needsHistoryRestore || monitorSandbox) { + const pollStartedAt = yield* Clock.currentTimeMillis; yield* watchEvents; if (active || needsHistoryRestore) yield* gate @@ -810,10 +830,33 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { yield* Effect.raceFirst( Effect.sleep(active ? "2 seconds" : "15 seconds"), Queue.take(wake), + ).pipe( + Effect.andThen( + Effect.gen(function* () { + // Stream notifications may reduce idle latency, but never amplify + // polling beyond one reconciliation per two seconds. + const elapsed = (yield* Clock.currentTimeMillis) - pollStartedAt; + if (elapsed < 2_000) yield* Effect.sleep(2_000 - elapsed); + }), + ), ); } - watching = false; - }).pipe(Effect.forkIn(scope)); + }).pipe( + Effect.onExit((exit) => + gate.withPermit( + Effect.gen(function* () { + if (streamFiber) yield* Fiber.interrupt(streamFiber); + streamFiber = undefined; + watching = false; + // A turn may arrive while the previous socket is closing. Starting + // and retiring the watcher share the turn gate, so its wake is not lost. + if (Exit.isSuccess(exit) && (active || needsHistoryRestore || monitorSandbox)) + yield* watch; + }), + ), + ), + Effect.forkIn(scope), + ); }); const policyHash = (policy: Adapter.ProviderAdapterV2RuntimePolicy) => wire( @@ -918,6 +961,11 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { "No durable cloud intent exists for this run. No task was resubmitted.", ); active = saved; + yield* emit({ + type: "provider_turn.updated", + driver, + providerTurn: saved.providerTurn, + }); binding = saved.binding ?? undefined; monitorSandbox = true; if ( @@ -953,9 +1001,9 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { "Plan mode is not yet supported for Kilo Cloud. No task was submitted.", ); const selectedPolicyHash = yield* policyHash(request.runtimePolicy); - const priorIntent = (yield* wire(options.journal.read)).findLast( - (entry) => entry.providerThread.id === request.providerThread.id, - ); + const priorIntent = (yield* wire( + options.journal.readThread(request.providerThread.id), + )).findLast((entry) => entry.providerThread.id === request.providerThread.id); if (priorIntent && priorIntent.policyHash !== selectedPolicyHash) return yield* error( "Cloud permissions changed. Start a separate cloud thread; the remote agent retains its original permissions.", @@ -987,6 +1035,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { nativeMetadata: { ...request.providerThread.nativeMetadata, continuationKey: options.continuationKey, + // Older correlations remain in their durable journal intents. turnCorrelations: { [messageId]: { messageId: request.message.messageId, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index a0bda9c42434..993afce88b18 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -3244,6 +3244,44 @@ it.effect("refreshes pull requests after a provider stream exits with an error", }), ); +it.effect( + "closes every event consumer after a failed cloud reattach without terminalizing the remote run", + () => + Effect.gen(function* () { + for (let attempt = 0; attempt < 3; attempt++) { + const started = yield* Deferred.make(); + const stopped = yield* Deferred.make(); + const result = yield* captureRootRunTermination({ + key: `cloud-reattach-failure-${attempt}`, + cloudReattach: true, + shouldFinalizeRun: () => Effect.succeed(true), + events: () => + Stream.unwrap(Deferred.succeed(started, undefined).pipe(Effect.as(Stream.never))).pipe( + Stream.ensuring(Deferred.succeed(stopped, undefined)), + ), + startTurn: (input) => + Deferred.await(started).pipe( + Effect.andThen( + Effect.fail( + new ProviderAdapterTurnStartError({ + driver: ProviderDriverKind.make("kilo-cloud"), + threadId: input.threadId, + providerThreadId: input.providerThread.id, + runId: input.runId, + cause: "temporary remote recovery failure", + }), + ), + ), + ), + }); + yield* Deferred.await(stopped); + assert.isTrue(result.startFailed); + assert.deepEqual(result.written, []); + assert.deepEqual(result.observed, []); + } + }), +); + it.effect("refreshes pull requests only once when startup failure closes its event stream", () => Effect.gen(function* () { const ingestionStarted = yield* Deferred.make(); @@ -3288,6 +3326,7 @@ it.effect("keeps completed runs completed when pull request refresh fails", () = function captureRootRunTermination(input: { readonly key: string; + readonly cloudReattach?: boolean; readonly shouldFinalizeRun: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly seedOpenSubagent?: boolean; @@ -3299,6 +3338,7 @@ function captureRootRunTermination(input: { }) { return Effect.gen(function* () { const ids = backgroundScenarioIds(input.key); + let startFailed = false; const providerInstanceId = ProviderInstanceId.make("codex"); const runningSubagent = makeRunOwnedSubagentFixture({ ids, @@ -3361,99 +3401,113 @@ function captureRootRunTermination(input: { yield* Effect.gen(function* () { const runExecution = yield* RunExecutionService.RunExecutionServiceV2; - yield* runExecution.startRootRun({ - commandId: CommandId.make(`command:${input.key}`), - appThread: { id: ids.threadId } as OrchestrationV2AppThread, - providerSessionId: ProviderSessionId.make(`session:${input.key}`), - session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, - events: Stream.empty, - subscribeEvents: Effect.succeed({ - events: - input.events?.(ids) ?? - Stream.fromIterable([ - ...(input.seedOpenSubagent - ? [ - { type: "subagent.updated", driver, subagent: runningSubagent } as const, - { - type: "node.updated", - driver, - node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), - } as const, - { - type: "turn_item.updated", - driver, - turnItem: makeRunOwnedSubagentTurnItemFixture({ - ids, - providerInstanceId, - childThreadId: ids.childThreadId, + yield* runExecution + .startRootRun({ + commandId: CommandId.make(`command:${input.key}`), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make(`session:${input.key}`), + reattach: input.cloudReattach ?? false, + session: { + driver: input.cloudReattach ? ProviderDriverKind.make("kilo-cloud") : driver, + providerSession: { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, + }, + events: Stream.empty, + subscribeEvents: Effect.succeed({ + events: + input.events?.(ids) ?? + Stream.fromIterable([ + ...(input.seedOpenSubagent + ? [ + { type: "subagent.updated", driver, subagent: runningSubagent } as const, + { + type: "node.updated", driver, - status: "running", - }), - } as const, - ] - : []), - rootTerminalEvent(ids, "interrupted"), - ] satisfies ReadonlyArray), - close: Deferred.succeed(ingestionDone, undefined), - }), - startTurn: input.startTurn ?? (() => Effect.void), - } as unknown as ProviderAdapterV2SessionRuntime, - run: { - id: ids.runId, - threadId: ids.threadId, - ordinal: 1, - providerInstanceId, - } as OrchestrationV2Run, - rootNode: { - id: ids.rootNodeId, - providerTurnId: ids.rootProviderTurnId, - } as OrchestrationV2ExecutionNode, - checkpointScope: { - id: CheckpointScopeId.make(`checkpoint-scope:${input.key}`), - } as OrchestrationV2CheckpointScope, - providerThread: { - id: ids.providerThreadId, - driver, - } as OrchestrationV2ProviderThread, - attempt: { - id: ids.attemptId, - providerTurnId: ids.rootProviderTurnId, - } as OrchestrationV2RunAttempt, - attemptId: ids.attemptId, - providerTurnOrdinal: 1, - shouldFinalizeRun: input.shouldFinalizeRun, - ...(input.hasUnpairedRunInterruptRequest === undefined - ? {} - : { - hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, + node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), + } as const, + { + type: "turn_item.updated", + driver, + turnItem: makeRunOwnedSubagentTurnItemFixture({ + ids, + providerInstanceId, + childThreadId: ids.childThreadId, + driver, + status: "running", + }), + } as const, + ] + : []), + rootTerminalEvent(ids, "interrupted"), + ] satisfies ReadonlyArray), + close: Deferred.succeed(ingestionDone, undefined), }), - message: { - messageId: MessageId.make(`message:${input.key}`), - text: "interrupt projection", - attachments: [], - createdBy: "user", - creationSource: "web", - }, - modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, - runtimePolicy: { - runtimeMode: "full-access", - interactionMode: "default", - cwd: process.cwd(), - approvalPolicy: "never", - sandboxPolicy: { - type: "readOnly", - access: { type: "fullAccess" }, - networkAccess: false, + startTurn: input.startTurn ?? (() => Effect.void), + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { + id: ids.rootNodeId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make(`checkpoint-scope:${input.key}`), + } as OrchestrationV2CheckpointScope, + providerThread: { + id: ids.providerThreadId, + driver, + } as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + shouldFinalizeRun: input.shouldFinalizeRun, + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, + }), + message: { + messageId: MessageId.make(`message:${input.key}`), + text: "interrupt projection", + attachments: [], + createdBy: "user", + creationSource: "web", }, - }, - }); + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }) + .pipe( + Effect.catch((error) => { + if (!input.cloudReattach) return Effect.fail(error); + startFailed = true; + return Effect.void; + }), + ); }).pipe(Effect.provide(testLayer)); yield* Deferred.await(ingestionDone); - return { written: yield* Ref.get(writtenItems), observed: yield* Ref.get(observed) }; + return { + written: yield* Ref.get(writtenItems), + observed: yield* Ref.get(observed), + startFailed, + }; }); } diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 3b22014436c6..afaa46b346d1 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1402,12 +1402,16 @@ export const layer: Layer.Layer< Effect.catchCause((cause) => input.session.driver === "kilo-cloud" && (input.reattach || Cause.hasInterruptsOnly(cause)) - ? Effect.fail( - new RunExecutionStartError({ - commandId: input.commandId, - runId: input.run.id, - cause, - }), + ? Fiber.interrupt(providerEventFiber).pipe( + Effect.andThen( + Effect.fail( + new RunExecutionStartError({ + commandId: input.commandId, + runId: input.run.id, + cause, + }), + ), + ), ) : Effect.logError("orchestration V2 provider turn start failed", { runId: input.run.id, diff --git a/apps/server/src/provider/Drivers/KiloCloudDriver.ts b/apps/server/src/provider/Drivers/KiloCloudDriver.ts index 0e9c24d33ece..f50f089f1155 100644 --- a/apps/server/src/provider/Drivers/KiloCloudDriver.ts +++ b/apps/server/src/provider/Drivers/KiloCloudDriver.ts @@ -74,6 +74,7 @@ export const KiloCloudDriver: ProviderDriver snapshot), refresh: Effect.gen(function* () { - const current = yield* account.load.pipe(Effect.orElseSucceed(() => null)); - if (current?.accountId === credentials.accountId) snapshot = authenticatedSnapshot; + const current = yield* account.load.pipe(Effect.result); + if (current._tag === "Success" && current.success.accountId === credentials.accountId) + snapshot = authenticatedSnapshot; else { + const temporarilyUnavailable = + current._tag === "Failure" && current.failure.reason === "invalid_response"; snapshot = { ...snapshot, status: "error", - message: - "Kilo login changed or is unavailable. Reconfigure this cloud account; existing remote tasks may still be running.", + message: temporarilyUnavailable + ? "Kilo account verification is temporarily unavailable. Retry later; existing remote tasks may still be running." + : "Kilo login changed or is unavailable. Reconfigure this cloud account; existing remote tasks may still be running.", }; } yield* PubSub.publish(changes, snapshot); diff --git a/apps/server/src/provider/Drivers/KiloDriver.ts b/apps/server/src/provider/Drivers/KiloDriver.ts index 52682b53d235..397a4e0bef9d 100644 --- a/apps/server/src/provider/Drivers/KiloDriver.ts +++ b/apps/server/src/provider/Drivers/KiloDriver.ts @@ -162,7 +162,7 @@ export const KiloDriver: ProviderDriver = { version: null, status: "warning", auth: { status: "unknown", profileId: input.config.accountId }, - message: "Kilo has not been checked yet.", + message: KiloRuntime.localExecutionBlocked.message, }, }), ); @@ -254,8 +254,7 @@ export const KiloDriver: ProviderDriver = { ...latest, status: "error" as const, installed: false, - message: - "Kilo could not be checked. Verify the binary, profile and CLI version 7.8.3.", + message: KiloRuntime.localExecutionBlocked.message, }), ), ); diff --git a/apps/server/src/provider/kilo/KiloCloudAccount.test.ts b/apps/server/src/provider/kilo/KiloCloudAccount.test.ts new file mode 100644 index 000000000000..96dcd897254c --- /dev/null +++ b/apps/server/src/provider/kilo/KiloCloudAccount.test.ts @@ -0,0 +1,62 @@ +// @effect-diagnostics nodeBuiltinImport:off - customer authentication over a loopback fixture. +import * as NodeHttp from "node:http"; +import * as NodeEvents from "node:events"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Schema from "effect/Schema"; +import * as Account from "./KiloCloudAccount.ts"; + +const encode = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +it.live("separates credential rejection, account support and temporary profile failures", () => + Effect.gen(function* () { + let responseStatus = 503; + let personal = true; + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cloud-account-" }); + yield* fs.makeDirectory(`${root}/data/kilo`, { recursive: true }); + const write = (key: string) => + fs.writeFileString(`${root}/data/kilo/auth.json`, encode({ kilo: { type: "api", key } })); + yield* write("synthetic-a"); + const server = yield* Effect.acquireRelease( + Effect.promise(async () => { + const server = NodeHttp.createServer((req, res) => { + res.writeHead(responseStatus, { "content-type": "application/json" }); + res.end( + encode({ + user: { id: req.headers.authorization === "Bearer synthetic-a" ? "a" : "b" }, + hasPersonalAccount: personal, + }), + ); + }); + server.listen(0, "127.0.0.1"); + await NodeEvents.once(server, "listening"); + return server; + }), + (server) => + Effect.promise( + () => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + ), + ); + const address = server.address(); + if (!address || typeof address === "string") return yield* Effect.die("No fixture address"); + const account = yield* Account.make(root, `http://127.0.0.1:${address.port}`); + assert.equal((yield* account.load.pipe(Effect.flip)).reason, "invalid_response"); + responseStatus = 401; + assert.equal((yield* account.load.pipe(Effect.flip)).reason, "rejected"); + responseStatus = 200; + personal = false; + assert.equal((yield* account.load.pipe(Effect.flip)).reason, "unsupported"); + personal = true; + assert.equal((yield* account.load).accountId, "a"); + yield* write("synthetic-b"); + assert.equal((yield* account.load).accountId, "b"); + yield* fs.writeFileString(`${root}/data/kilo/auth.json`, "malformed"); + assert.equal((yield* account.load.pipe(Effect.flip)).reason, "rejected"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/provider/kilo/KiloCloudAccount.ts b/apps/server/src/provider/kilo/KiloCloudAccount.ts index ecc774932941..d346435f6204 100644 --- a/apps/server/src/provider/kilo/KiloCloudAccount.ts +++ b/apps/server/src/provider/kilo/KiloCloudAccount.ts @@ -6,6 +6,8 @@ import * as Schema from "effect/Schema"; import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; import { KiloCloudError } from "./KiloCloudClient.ts"; +const isKiloCloudError = Schema.is(KiloCloudError); + const Auth = Schema.Struct({ kilo: Schema.Union([ Schema.Struct({ type: Schema.Literal("oauth"), access: Schema.NonEmptyString }), @@ -21,24 +23,41 @@ const decodeAuth = Schema.decodeUnknownEffect(Schema.fromJsonString(Auth)); const decodeProfile = Schema.decodeUnknownEffect(Profile); /** Reads only the selected official CLI profile. Login and token refresh remain Kilo's job. */ -export const make = Effect.fn("KiloCloudAccount.make")(function* (profileDirectory: string) { +export const make = Effect.fn("KiloCloudAccount.make")(function* ( + profileDirectory: string, + origin = "https://app.kilo.ai", +) { + if (origin !== "https://app.kilo.ai" && !/^http:\/\/127\.0\.0\.1:\d+$/.test(origin)) + return yield* new KiloCloudError({ operation: "authentication", reason: "rejected" }); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; let cached: { token: Redacted.Redacted; accountId: string } | undefined; const load = Effect.gen(function* () { const saved = yield* decodeAuth( - yield* fs.readFileString(path.join(profileDirectory, "data", "kilo", "auth.json")), + yield* fs + .readFileString(path.join(profileDirectory, "data", "kilo", "auth.json")) + .pipe( + Effect.mapError( + () => new KiloCloudError({ operation: "credentials", reason: "rejected" }), + ), + ), + ).pipe( + Effect.mapError(() => new KiloCloudError({ operation: "credentials", reason: "rejected" })), ); const token = saved.kilo.type === "oauth" ? saved.kilo.access : saved.kilo.key; if (cached && Redacted.value(cached.token) === token) return cached; const client = yield* HttpClient.HttpClient; const response = yield* client.execute( - HttpClientRequest.get("https://app.kilo.ai/api/profile", { + HttpClientRequest.get(`${origin}/api/profile`, { headers: { authorization: `Bearer ${token}` }, }), ); if (response.status !== 200) - return yield* new KiloCloudError({ operation: "authentication", reason: "rejected" }); + return yield* new KiloCloudError({ + operation: "authentication", + reason: + response.status === 401 || response.status === 403 ? "rejected" : "invalid_response", + }); const profile = yield* decodeProfile(yield* response.json); if (!profile.hasPersonalAccount) return yield* new KiloCloudError({ operation: "personal-account", reason: "unsupported" }); @@ -49,7 +68,11 @@ export const make = Effect.fn("KiloCloudAccount.make")(function* (profileDirecto Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), Effect.provide(FetchHttpClient.layer), Effect.timeout("15 seconds"), - Effect.mapError(() => new KiloCloudError({ operation: "authentication", reason: "rejected" })), + Effect.mapError((cause) => + isKiloCloudError(cause) + ? cause + : new KiloCloudError({ operation: "authentication", reason: "invalid_response" }), + ), ); return { load }; }); diff --git a/apps/server/src/provider/kilo/KiloCloudClient.test.ts b/apps/server/src/provider/kilo/KiloCloudClient.test.ts index 4c8b3a0370e7..6cbb2a8cd3ac 100644 --- a/apps/server/src/provider/kilo/KiloCloudClient.test.ts +++ b/apps/server/src/provider/kilo/KiloCloudClient.test.ts @@ -87,16 +87,19 @@ describe("Kilo customer Cloud Agent boundary", () => { expect(failure.messageId).toBe(messageId); expect(accepted).toBe(1); }); - it.each([500, 503, 307])("never follows or retries mutation status %i", async (status) => { - let requests = 0; - const cloud = await client((_request, response) => { - requests++; - response.writeHead(status, { location: "/other" }); - response.end(); - }); - expect((await run(cloud.start(start).pipe(Effect.flip))).reason).toBe("admission_unknown"); - expect(requests).toBe(1); - }); + it.each([408, 409, 500, 503, 307])( + "never follows or retries mutation status %i", + async (status) => { + let requests = 0; + const cloud = await client((_request, response) => { + requests++; + response.writeHead(status, { location: "/other" }); + response.end(); + }); + expect((await run(cloud.start(start).pipe(Effect.flip))).reason).toBe("admission_unknown"); + expect(requests).toBe(1); + }, + ); it("rejects a mismatched session on send and a mismatched message on result", async () => { const cloud = await client((request, response) => json( diff --git a/apps/server/src/provider/kilo/KiloCloudClient.ts b/apps/server/src/provider/kilo/KiloCloudClient.ts index 1259c981f353..ccf0251a1bcc 100644 --- a/apps/server/src/provider/kilo/KiloCloudClient.ts +++ b/apps/server/src/provider/kilo/KiloCloudClient.ts @@ -12,6 +12,7 @@ export class KiloCloudError extends Schema.TaggedError()("KiloCl operation: Schema.String, reason: Schema.Literals([ "rejected", + "not_found", "admission_unknown", "invalid_response", "wrong_owner", @@ -91,7 +92,7 @@ export const make = (input: { mutation ? HttpClientRequest.bodyText(req, encode(body), "application/json") : req, ); if (response.status < 200 || response.status >= 300) - return yield* response.status >= 500 + return yield* response.status >= 500 || response.status === 408 || response.status === 409 ? uncertain() : new KiloCloudError({ operation, reason: "rejected", messageId }); const bytes = yield* response.stream.pipe( diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts index 8bd19b50f68d..f7822d1750dd 100644 --- a/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts @@ -73,6 +73,152 @@ function json(response: NodeHttp.ServerResponse, data: unknown) { response.end(JSON.stringify({ result: { data } })); } describe("Kilo personal Cloud control-plane customer API", () => { + it("continues an uncertain-admission scan across read budgets and cursor pages without resubmitting", async () => { + const reads: string[] = []; + const cursors: Array = []; + const candidates = Array.from({ length: 101 }, (_, index) => ({ + session_id: `ses_candidate${index}`, + cloud_agent_session_id: `workspace_12345678-1234-1234-1234-${String(index).padStart(12, "0")}`, + })); + const { client } = await server((req, res) => { + expect(req.method).toBe("GET"); + const url = new URL(req.url!, "http://localhost"); + const input = JSON.parse(url.searchParams.get("input")!) as Record; + if (url.pathname.endsWith("cliSessionsV2.list")) { + cursors.push(input.cursor ?? null); + expect(input.organizationId).toBeNull(); + return json(res, { + cliSessions: input.cursor ? candidates.slice(100) : candidates.slice(0, 100), + nextCursor: input.cursor ? null : "2026-10-01T00:00:00.000Z", + }); + } + const candidate = candidates.find( + (item) => item.cloud_agent_session_id === input.cloudAgentSessionId, + )!; + reads.push(candidate.session_id); + return json(res, { + ...session, + sessionId: candidate.cloud_agent_session_id, + kiloSessionId: candidate.session_id, + initialMessageId: + candidate === candidates[100] ? messageId : "msg_00000000000000000000000000", + }); + }); + for (let index = 0; index < 4; index++) + expect(await run(client.findAdmission(binding.repository, messageId))).toBeNull(); + expect(await run(client.findAdmission(binding.repository, messageId))).toEqual({ + cloudAgentSessionId: candidates[100]!.cloud_agent_session_id, + kiloSessionId: candidates[100]!.session_id, + }); + expect(reads).toHaveLength(101); + expect(new Set(reads).size).toBe(101); + expect(cursors).toEqual([null, "2026-10-01T00:00:00.000Z"]); + }); + it("does not let a transient or deleted candidate starve a later personal session", async () => { + const reads: string[] = []; + const missing = "workspace_00000000-0000-0000-0000-000000000000"; + let first = true; + const { client } = await server((req, res) => { + const url = new URL(req.url!, "http://localhost"); + const input = JSON.parse(url.searchParams.get("input")!) as Record; + if (url.pathname.endsWith("cliSessionsV2.list")) + return json(res, { + cliSessions: [ + { session_id: "ses_deleted", cloud_agent_session_id: missing }, + { session_id: session.kiloSessionId, cloud_agent_session_id: session.sessionId }, + ], + nextCursor: null, + }); + reads.push(input.cloudAgentSessionId!); + if (input.cloudAgentSessionId === missing) { + res.writeHead(first ? 503 : 404); + first = false; + res.end(); + return; + } + return json(res, session); + }); + expect(await run(client.findAdmission(binding.repository, messageId))).toBeNull(); + expect(await run(client.findAdmission(binding.repository, messageId))).toEqual({ + cloudAgentSessionId: session.sessionId, + kiloSessionId: session.kiloSessionId, + }); + expect(reads).toEqual([missing, session.sessionId, missing]); + }); + it("searches later pages despite a persistently unavailable candidate and waits before binding", async () => { + let unavailable = true; + let pageTwoRead = false; + const missing = "workspace_00000000-0000-0000-0000-000000000000"; + const { client } = await server((req, res) => { + const url = new URL(req.url!, "http://localhost"); + const input = JSON.parse(url.searchParams.get("input")!) as Record; + if (url.pathname.endsWith("cliSessionsV2.list")) { + if (input.cursor) pageTwoRead = true; + return json( + res, + input.cursor + ? { + cliSessions: [ + { session_id: session.kiloSessionId, cloud_agent_session_id: session.sessionId }, + ], + nextCursor: null, + } + : { + cliSessions: [{ session_id: "ses_unavailable", cloud_agent_session_id: missing }], + nextCursor: "2026-10-01T00:00:00.000Z", + }, + ); + } + if (input.cloudAgentSessionId === missing) { + res.writeHead(unavailable ? 503 : 404); + res.end(); + return; + } + return json(res, session); + }); + expect(await run(client.findAdmission(binding.repository, messageId))).toBeNull(); + expect(pageTwoRead).toBe(true); + expect(await run(client.findAdmission(binding.repository, messageId))).toBeNull(); + unavailable = false; + expect(await run(client.findAdmission(binding.repository, messageId))).toEqual({ + cloudAgentSessionId: session.sessionId, + kiloSessionId: session.kiloSessionId, + }); + }); + it("rejects ambiguous admission identities and repeating cursors without a mutation", async () => { + let repeated = false; + const { client } = await server((req, res) => { + expect(req.method).toBe("GET"); + const url = new URL(req.url!, "http://localhost"); + if (url.pathname.endsWith("cliSessionsV2.list")) + return json(res, { + cliSessions: repeated + ? [] + : [ + { session_id: session.kiloSessionId, cloud_agent_session_id: session.sessionId }, + { + session_id: "ses_second", + cloud_agent_session_id: "workspace_00000000-0000-0000-0000-000000000000", + }, + ], + nextCursor: repeated ? "2026-10-01T00:00:00.000Z" : null, + }); + const input = JSON.parse(url.searchParams.get("input")!) as Record; + return json(res, { + ...session, + sessionId: input.cloudAgentSessionId, + kiloSessionId: + input.cloudAgentSessionId === session.sessionId ? session.kiloSessionId : "ses_second", + }); + }); + expect( + (await run(client.findAdmission(binding.repository, messageId).pipe(Effect.flip))).reason, + ).toBe("wrong_owner"); + repeated = true; + expect( + (await run(client.findAdmission(binding.repository, messageId).pipe(Effect.flip))).reason, + ).toBe("invalid_response"); + }); it("authenticates a customer WebSocket, resumes its cursor and rejects a foreign session event", async () => { const expiresAt = await run(Clock.currentTimeMillis); const { client, httpServer } = await server((req, res) => { @@ -127,6 +273,7 @@ describe("Kilo personal Cloud control-plane customer API", () => { ); expect(received).toEqual([38]); expect(error.operation).toBe("events"); + expect(error.reason).toBe("wrong_owner"); }); it.each([ "varCount", diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.ts index 9964ae9adc57..2d29d2ee3a05 100644 --- a/apps/server/src/provider/kilo/KiloCloudWebClient.ts +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.ts @@ -210,7 +210,11 @@ export const make = (options: { if (response.status < 200 || response.status >= 300) return yield* response.status >= 500 || response.status === 408 || response.status === 409 ? uncertain() - : failure(operation, "rejected", messageId); + : failure( + operation, + !mutation && response.status === 404 ? "not_found" : "rejected", + messageId, + ); const chunks: Uint8Array[] = []; let size = 0; yield* response.stream.pipe( @@ -300,6 +304,19 @@ export const make = (options: { return yield* failure("profile-preflight", "rejected"); } }); + // Keep progress across the adapter's bounded reconcile calls. A timed-out GET + // retries only that read, never the paid admission or the whole first page. + const admissionScans = new Map< + string, + { + cursor?: string | undefined; + pending: Array<{ session_id: string; cloud_agent_session_id: string | null }>; + deferred: Array<{ session_id: string; cloud_agent_session_id: string | null }>; + loaded: boolean; + seenCursors: Set; + matches: Map; + } + >(); return { getSession, /** Short-lived customer tickets only. A closed socket has no stop semantics. */ @@ -342,40 +359,101 @@ export const make = (options: { }), ).pipe( Stream.scoped, - Stream.mapError(() => failure("events", "invalid_response")), + Stream.mapError((cause) => + isCloudError(cause) ? cause : failure("events", "invalid_response"), + ), ), // Recovery only: enumerate access-checked metadata and correlate the persisted // initial message, never replay a paid prepare after a lost response. findAdmission: (repository: string, initialMessageId: string) => Effect.gen(function* () { - const page = yield* request( - "cliSessionsV2.list", - { gitUrl: `https://github.com/${repository}`, limit: 100 }, - Schema.Struct({ - cliSessions: Schema.Array( + const key = `${repository}\0${initialMessageId}`; + let scan = admissionScans.get(key); + if (!scan) { + scan = { + pending: [], + deferred: [], + loaded: false, + seenCursors: new Set(), + matches: new Map(), + }; + admissionScans.set(key, scan); + } + // At most 25 candidate reads per call. Later polls continue this scan. + for (let budget = 25; budget > 0; budget--) { + if (!scan.pending.length) { + if (scan.loaded && !scan.cursor) { + if (scan.deferred.length) { + scan.pending = scan.deferred; + scan.deferred = []; + return null; + } + admissionScans.delete(key); + if (scan.matches.size > 1) + return yield* failure("reconcile-admission", "wrong_owner"); + return [...scan.matches.values()][0] ?? null; + } + const page = yield* request( + "cliSessionsV2.list", + { + gitUrl: `https://github.com/${repository}`, + limit: 100, + orderBy: "created_at", + organizationId: null, + ...(scan.cursor ? { cursor: scan.cursor } : {}), + }, Schema.Struct({ - session_id: NativeId, - cloud_agent_session_id: Schema.NullOr(Schema.String), + cliSessions: Schema.Array( + Schema.Struct({ + session_id: NativeId, + cloud_agent_session_id: Schema.NullOr(Schema.String), + }), + ), + nextCursor: Schema.NullOr(Schema.String), + }), + ); + if (page.nextCursor && scan.seenCursors.has(page.nextCursor)) + return yield* failure("reconcile-admission", "invalid_response"); + if (page.nextCursor) scan.seenCursors.add(page.nextCursor); + scan.cursor = page.nextCursor ?? undefined; + scan.loaded = true; + scan.pending = page.cliSessions.filter((candidate) => + candidate.cloud_agent_session_id?.startsWith("workspace_"), + ); + if (!scan.pending.length) continue; + } + const candidate = scan.pending.shift()!; + const deferred = scan.deferred; + const session = yield* getSession(candidate.cloud_agent_session_id!).pipe( + Effect.catchTag("KiloCloudError", (error) => { + if (error.reason === "not_found") return Effect.succeed(null); + if (error.reason === "invalid_response") + return Effect.sync(() => { + deferred.push(candidate); + return null; + }); + return Effect.fail(error); + }), + // Preserve and rotate an interrupted/unavailable read. A stale first + // candidate must not starve all subsequent candidates on every poll. + Effect.onError(() => + Effect.sync(() => { + deferred.push(candidate); }), ), - }), - ); - const matches: Array = []; - for (const candidate of page.cliSessions) { - if (!candidate.cloud_agent_session_id?.startsWith("workspace_")) continue; - const session = yield* getSession(candidate.cloud_agent_session_id); + ); + if (!session) continue; if ( session.initialMessageId === initialMessageId && session.githubRepo === repository && session.kiloSessionId === candidate.session_id ) - matches.push({ + scan.matches.set(session.sessionId, { cloudAgentSessionId: session.sessionId, kiloSessionId: session.kiloSessionId, }); } - if (matches.length > 1) return yield* failure("reconcile-admission", "wrong_owner"); - return matches[0] ?? null; + return null; }), /** Admission is paid. Persist operationKey and initialMessageId before calling; no retries here. */ prepare: (input: { diff --git a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts index 68ed14fb0d81..1142b2693e87 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts @@ -29,7 +29,8 @@ const environment = { KILO_DISABLE_PROJECT_CONFIG: "1", }; -describe.runIf(binary !== undefined)("KiloRuntime native lifecycle", () => { +// Historical native conformance coverage. Re-enable only after an audited MCP runtime fix. +describe.skip("KiloRuntime native lifecycle", () => { it.live( "does not execute repository or external plugins before session permissions", () => diff --git a/apps/server/src/provider/kilo/KiloRuntime.safety.test.ts b/apps/server/src/provider/kilo/KiloRuntime.safety.test.ts new file mode 100644 index 000000000000..aabdc4dc1038 --- /dev/null +++ b/apps/server/src/provider/kilo/KiloRuntime.safety.test.ts @@ -0,0 +1,99 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Schema from "effect/Schema"; +import * as Path from "effect/Path"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import * as KiloRuntime from "./KiloRuntime.ts"; + +const encode = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); + +it.live("rejects every local open before executing the binary or changing config", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-safety-" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + let spawned = 0; + const observedSpawner = { + ...spawner, + spawn: (...args: Parameters) => { + spawned++; + return spawner.spawn(...args); + }, + }; + const profile = path.join(root, "profile"); + const fixtures = [ + ".kilo/mcp.json", + ".kilocode/mcp.json", + "profile/config/kilo/kilo.json", + ".kilo/kilo.json", + ]; + const sources = new Map(); + for (const [index, file] of fixtures.entries()) { + const target = path.join(root, file); + const command = [ + "node", + "-e", + `require('node:fs').writeFileSync(${encode(path.join(root, "marker"))}, 'executed')`, + ]; + const contents = encode( + file.endsWith("mcp.json") + ? { + mcpServers: { + [`new-server-${index}`]: { command: command[0], args: command.slice(1) }, + }, + } + : { + mcp: { [`new-server-${index}`]: { type: "local", command } }, + }, + ); + yield* fs.makeDirectory(path.dirname(target), { recursive: true }); + yield* fs.writeFileString(target, contents); + sources.set(target, contents); + } + const runtime = yield* KiloRuntime.make({ + instanceId: "safety-a", + binaryPath: process.env.KILO_BIN ?? "kilo", + profileDirectory: profile, + environment: { + PATH: process.env.PATH, + KILO_PURE: "0", + KILO_DISABLE_PROJECT_CONFIG: "0", + KILO_PLATFORM: "vscode", + KILOCODE_FEATURE: "daemon", + }, + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, observedSpawner)); + for (const directory of [root, path.join(root, "nested"), root]) { + const failure = yield* runtime.open(directory).pipe(Effect.flip); + assert.equal(failure.operation, "runtime-safety"); + } + // A later new server name and account change cannot open a second entry path. + yield* fs.writeFileString( + path.join(root, ".kilocode/mcp.json"), + '{"mcpServers":{"later-server":{"url":"http://127.0.0.1:9"}}}', + ); + sources.delete(path.join(root, ".kilocode/mcp.json")); + const failure = yield* runtime.open(root).pipe(Effect.flip); + assert.equal(failure.operation, "runtime-safety"); + assert.equal(spawned, 0); + assert.isFalse(yield* fs.exists(path.join(root, "marker"))); + for (const [target, contents] of sources) + assert.equal(yield* fs.readFileString(target), contents); + // Credential selection still works, without starting a native process. + const authDir = path.join(profile, "data/kilo"); + yield* fs.makeDirectory(authDir, { recursive: true }); + yield* fs.writeFileString( + path.join(authDir, "auth.json"), + '{"kilo":{"type":"api","key":"synthetic-a"}}', + ); + const first = yield* KiloRuntime.readAuth(profile, {}); + yield* fs.writeFileString( + path.join(authDir, "auth.json"), + '{"kilo":{"type":"api","key":"synthetic-b"}}', + ); + assert.notEqual(first, yield* KiloRuntime.readAuth(profile, {})); + assert.equal(yield* KiloRuntime.readAuth(profile, { KILO_AUTH_CONTENT: first }), first); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/provider/kilo/KiloRuntime.ts b/apps/server/src/provider/kilo/KiloRuntime.ts index 19bb626fd257..2b1c391059a0 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.ts @@ -81,6 +81,17 @@ export const readAuth = Effect.fn("KiloRuntime.readAuth")(function* ( ); }); +// No released runtime has a verified pre-connect MCP policy boundary. This gate +// deliberately has no environment/config override. Restore execution only with +// an audited dependency and real-process startup/reload/reconnect tests. +export const localExecutionBlocked = new KiloRuntimeError({ + operation: "runtime-safety", + detail: + "Local Kilo execution is disabled: CLI 7.8.3 can start MCP commands and connections before approval. A verified runtime fix is required.", +}); +const requireSafeRuntime: Effect.Effect = + Effect.fail(localExecutionBlocked); + /** Every open owns a process. Registry replacement closes the old account's process scopes. */ export const make = Effect.fn("KiloRuntime.make")(function* (input: { readonly instanceId: string; @@ -146,6 +157,7 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { // Readiness output can contain project/plugin diagnostics; never include it in client errors. return KiloRuntime.of({ open: Effect.fn("KiloRuntime.open")(function* (directory) { + yield* requireSafeRuntime; if (closed) return yield* new KiloRuntimeError({ operation: "open", diff --git a/apps/server/src/textGeneration/KiloTextGeneration.ts b/apps/server/src/textGeneration/KiloTextGeneration.ts index 75481b55b804..0c27abbe9864 100644 --- a/apps/server/src/textGeneration/KiloTextGeneration.ts +++ b/apps/server/src/textGeneration/KiloTextGeneration.ts @@ -5,9 +5,13 @@ import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import { resolveAttachmentPath } from "../attachmentStore.ts"; import { toOpenCodeFileParts } from "../provider/opencodeRuntime.ts"; -import type * as KiloRuntime from "../provider/kilo/KiloRuntime.ts"; +import * as KiloRuntime from "../provider/kilo/KiloRuntime.ts"; import { makeOpenCodeOperations, type OpenCodeJsonRunner } from "./OpenCodeTextGeneration.ts"; +const isKiloRuntimeError = Schema.is(KiloRuntime.KiloRuntimeError); + +const isTextGenerationError = Schema.is(TextGenerationError); + /** Only prompt construction is shared. Protocol, credentials and lifetime belong to Kilo. */ export function make(runtime: KiloRuntime.KiloRuntime["Service"], attachmentsDir?: string) { const run: OpenCodeJsonRunner = (input) => @@ -56,12 +60,16 @@ export function make(runtime: KiloRuntime.KiloRuntime["Service"], attachmentsDir ); }).pipe( Effect.scoped, - Effect.mapError( - () => - new TextGenerationError({ - operation: input.operation, - detail: "Kilo text generation failed. The request was not retried.", - }), + Effect.mapError((cause) => + isTextGenerationError(cause) + ? cause + : new TextGenerationError({ + operation: input.operation, + detail: isKiloRuntimeError(cause) + ? cause.message + : "Kilo text generation failed. The request was not retried.", + cause, + }), ), ); return makeOpenCodeOperations(run); diff --git a/apps/web/src/components/ChatMarkdown.test.tsx b/apps/web/src/components/ChatMarkdown.test.tsx index 5585dc3ddf40..c99902c75f3f 100644 --- a/apps/web/src/components/ChatMarkdown.test.tsx +++ b/apps/web/src/components/ChatMarkdown.test.tsx @@ -1,10 +1,11 @@ -import { EnvironmentId } from "@t3tools/contracts"; +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; import { act, type ComponentProps, type ReactNode } from "react"; import { renderToStaticMarkup } from "react-dom/server"; import { create, type ReactTestRenderer } from "react-test-renderer"; import { describe, expect, it, vi } from "vite-plus/test"; import { getSyntaxHighlighterPromise } from "../lib/syntaxHighlighting"; +import { useRightPanelStore } from "../rightPanelStore"; import { GitHubIcon } from "./Icons"; import { Button } from "./ui/button"; import { setMarkdownTaskChecked } from "./files/filePreviewMode"; @@ -60,6 +61,7 @@ vi.mock("~/lib/openPullRequestLink", () => ({ })); import ChatMarkdown, { + ChatMarkdownLocalWorkspaceContext, canUseMarkdownFileShellActions, hasMarkdownFilePrimaryAction, shouldUseMarkdownFileBrowserPrimaryAction, @@ -73,6 +75,70 @@ function codeButton(renderer: ReactTestRenderer, label: string) { return button.props as ComponentProps; } +describe("cloud Markdown workspace boundary", () => { + it("opens a local file in a local thread but never exposes host file or shell actions in cloud text", async () => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + const threadRef = { + environmentId: EnvironmentId.make("workspace-boundary"), + threadId: ThreadId.make("workspace-boundary"), + }; + const text = + "[File](/tmp/project/src/secret.txt) and [Web](https://example.com)\n\n```bash\necho harmless\n```"; + const shell = vi.fn(); + let renderer: ReactTestRenderer | undefined; + const render = (allowed: boolean) => ( + + + + ); + try { + await act(async () => { + renderer = create(render(true)); + }); + const link = renderer!.root + .findAllByType("a") + .find((node) => node.props.href === "/tmp/project/src/secret.txt"); + expect(link).toBeDefined(); + await act(async () => { + link!.props.onClick({ preventDefault() {}, stopPropagation() {} }); + }); + const opened = Object.values(useRightPanelStore.getState().byThreadKey).flatMap( + (entry) => entry.surfaces, + ); + expect( + opened.some( + (surface) => surface.kind === "file" && surface.relativePath === "src/secret.txt", + ), + ).toBe(true); + await act(async () => { + renderer!.update(render(false)); + }); + expect( + renderer!.root + .findAllByType("a") + .some((node) => node.props.href === "/tmp/project/src/secret.txt"), + ).toBe(false); + expect( + renderer!.root.findAllByType("a").some((node) => node.props.href === "https://example.com"), + ).toBe(true); + expect( + renderer!.root + .findAllByType(Button) + .some((node) => node.props["aria-label"] === "Run in terminal"), + ).toBe(false); + expect(shell).not.toHaveBeenCalled(); + } finally { + if (renderer) await act(async () => renderer!.unmount()); + vi.unstubAllGlobals(); + } + }); +}); + describe("ChatMarkdown context references", () => { it("renders text and image references through the chip renderer, with readable fallback", async () => { vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); diff --git a/apps/web/src/components/ChatMarkdown.tsx b/apps/web/src/components/ChatMarkdown.tsx index ea02607cc115..4b6e3740037d 100644 --- a/apps/web/src/components/ChatMarkdown.tsx +++ b/apps/web/src/components/ChatMarkdown.tsx @@ -1417,6 +1417,7 @@ const CHAT_MARKDOWN_WORKSPACE_IMAGE_CLASS_NAME = cn( CHAT_MARKDOWN_MEDIA_LAYOUT_CLASS_NAME, CHAT_MARKDOWN_MEDIA_FRAME_CLASS_NAME, ); +export const ChatMarkdownLocalWorkspaceContext = React.createContext(true); const MarkdownLinkContext = React.createContext(false); function expandableMarkdownImageProps( @@ -1658,8 +1659,23 @@ function ChatMarkdownVideo(props: { ); } +export const ChatMarkdownAssetImage = memo(function ChatMarkdownAssetImage( + props: ComponentProps, +) { + const allowed = use(ChatMarkdownLocalWorkspaceContext); + return allowed ? ( + + ) : ( + + ); +}); + /** Environment-hosted media loads through an exact-file signed asset URL. */ -export const ChatMarkdownAssetImage = memo(function ChatMarkdownAssetImage(props: { +const LocalChatMarkdownAssetImage = memo(function LocalChatMarkdownAssetImage(props: { readonly environmentId: EnvironmentId; readonly resource: Extract< AssetResource, @@ -2337,6 +2353,7 @@ function useChatMarkdownState({ headingLevelOffset = 0, githubMedia = false, }: ChatMarkdownProps) { + const localWorkspaceEnabled = use(ChatMarkdownLocalWorkspaceContext); const { resolvedTheme } = useTheme(); const [localMediaPreview, setLocalMediaPreview] = useState(null); const markdownRef = useRef(null); @@ -2646,6 +2663,7 @@ function useChatMarkdownState({ ); const fileLinkChip = useCallback( (fileLinkMeta: MarkdownFileLinkMeta, copyMarkdown: string, mediaSource?: string) => { + if (!localWorkspaceEnabled) return {fileLinkMeta.displayPath}; const parentSuffix = fileLinkParentSuffixByPath.get( fileLinkMeta.filePath.replaceAll("\\", "/"), ); @@ -2706,6 +2724,7 @@ function useChatMarkdownState({ ); }, [ + localWorkspaceEnabled, canUseShellActions, fileLinkParentSuffixByPath, openFileInPanel, @@ -3364,7 +3383,23 @@ function ChatMarkdown({ markdownUrlTransform, localMediaPreview, setLocalMediaPreview, - } = useChatMarkdownState({ text, ...props }); + } = useChatMarkdownState({ + text, + ...props, + ...(!use(ChatMarkdownLocalWorkspaceContext) + ? { + cwd: undefined, + threadRef: undefined, + environmentId: undefined, + imageBaseDir: undefined, + pullRequestPanelRef: undefined, + renderContextReference: undefined, + onRunShellCommand: undefined, + onUseArtifactTemplate: undefined, + githubMedia: false, + } + : {}), + }); const incrementalParsing = props.isStreaming === true && extraRemarkPlugins.length === 0 && diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index daf1e5f2ab1b..017fe116852f 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -413,6 +413,7 @@ import { DraftHeroHeadline } from "./chat/DraftHeroHeadline"; import { ExpandedImageDialog } from "./chat/ExpandedImageDialog"; import { PullRequestThreadDialog } from "./PullRequestThreadDialog"; import type { AssistantCitationRequest } from "./chat/AssistantCitationSource"; +import { ChatMarkdownLocalWorkspaceContext } from "./ChatMarkdown"; import { MessagesTimeline, type MessagesTimelineHistoryControls } from "./chat/MessagesTimeline"; import { ProviderSubagentBar } from "./chat/ProviderSubagentBar"; import { getTriggerDisplayModelName } from "./chat/providerIconUtils"; @@ -3148,6 +3149,7 @@ export default function ChatView(props: ChatViewProps) { providerStatuses.find((status) => status.instanceId === activeRuntime?.providerInstanceId) ?? activeProviderStatus; const supportsConversationRollback = + !isCloudThread && conversationProviderStatus !== null && conversationProviderStatus.supportsConversationRollback !== false; const phase = derivePhase(activeRuntime); @@ -3934,13 +3936,14 @@ export default function ChatView(props: ChatViewProps) { : JSON.stringify([itemId, latestCheckpointCompletedAt]); }, [serverVisibleTurnItems, turnDiffSummaries]); - const gitCwd = activeProject - ? projectScriptCwd({ - project: { cwd: activeProject.workspaceRoot }, - worktreePath: activeThread?.worktreePath ?? null, - }) - : null; - const gitStatusCwd = activeThread?.worktreePath ?? gitCwd; + const gitCwd = + !isCloudThread && activeProject + ? projectScriptCwd({ + project: { cwd: activeProject.workspaceRoot }, + worktreePath: activeThread?.worktreePath ?? null, + }) + : null; + const gitStatusCwd = isCloudThread ? null : (activeThread?.worktreePath ?? gitCwd); const gitStatusQuery = useEnvironmentQuery( gitStatusCwd === null ? null @@ -5017,7 +5020,7 @@ export default function ChatView(props: ChatViewProps) { ); const createBrowserSurface = useCallback( (profileId?: string) => { - if (!activeThreadRef) return; + if (isCloudThread || !activeThreadRef) return; void addBrowserSurface({ threadRef: activeThreadRef, openPreview, @@ -5036,7 +5039,7 @@ export default function ChatView(props: ChatViewProps) { } }); }, - [activeThreadRef, openPreview], + [isCloudThread, activeThreadRef, openPreview], ); const addDiffSurface = useCallback(() => { if (isCloudThread || !activeThreadRef || !isServerThread || !isGitRepo) return; @@ -5060,21 +5063,21 @@ export default function ChatView(props: ChatViewProps) { const pullRequestsSurfaceAvailable = isServerThread && supportsThreadPullRequests && visiblePullRequestCount > 0; const addPullRequestsSurface = useCallback(() => { - if (!activeThreadRef || !pullRequestsSurfaceAvailable) return; + if (isCloudThread || !activeThreadRef || !pullRequestsSurfaceAvailable) return; useRightPanelStore.getState().open(activeThreadRef, "pull-requests"); - }, [activeThreadRef, pullRequestsSurfaceAvailable]); + }, [isCloudThread, activeThreadRef, pullRequestsSurfaceAvailable]); const { state: deviceState, loaded: deviceStateLoaded } = useDeviceState( activeThreadRef?.environmentId ?? null, ); const [deviceSetupThread, setDeviceSetupThread] = useState(null); const addDeviceSurface = useCallback(() => { - if (!activeThreadRef) return; + if (isCloudThread || !activeThreadRef) return; if (!deviceState.onboardingCompleted || deviceState.hostStatus === "disabled") { setDeviceSetupThread(activeThreadRef); return; } useRightPanelStore.getState().open(activeThreadRef, "device"); - }, [activeThreadRef, deviceState.onboardingCompleted, deviceState.hostStatus]); + }, [isCloudThread, activeThreadRef, deviceState.onboardingCompleted, deviceState.hostStatus]); // A device the agent opens floats over chat like an agent-driven browser, // or becomes a panel tab when floating previews are off. Sessions opened by // another client arrive the same way; sheet layouts get neither. The first @@ -6485,7 +6488,7 @@ export default function ChatView(props: ChatViewProps) { }); const localCheckoutBranchMismatch = useMemo( () => - isServerThread + isServerThread && !isCloudThread ? resolveLocalCheckoutBranchMismatch({ effectiveEnvMode: envMode, activeWorktreePath, @@ -6493,7 +6496,14 @@ export default function ChatView(props: ChatViewProps) { currentGitBranch: gitStatusQuery.data?.refName ?? null, }) : null, - [activeThreadBranch, activeWorktreePath, envMode, gitStatusQuery.data?.refName, isServerThread], + [ + activeThreadBranch, + activeWorktreePath, + envMode, + gitStatusQuery.data?.refName, + isServerThread, + isCloudThread, + ], ); const publishComposerOverlayHeight = useCallback( (height: number) => { @@ -6635,10 +6645,15 @@ export default function ChatView(props: ChatViewProps) { ); }, [activeThreadReferenceCopyTarget]); const addPullRequestSurface = useCallback(() => { - if (!supportsPullRequests || activeThreadRef === null || linkedThreadPullRequest === null) + if ( + isCloudThread || + !supportsPullRequests || + activeThreadRef === null || + linkedThreadPullRequest === null + ) return; useRightPanelStore.getState().openPullRequest(activeThreadRef, linkedThreadPullRequest); - }, [activeThreadRef, linkedThreadPullRequest, supportsPullRequests]); + }, [isCloudThread, activeThreadRef, linkedThreadPullRequest, supportsPullRequests]); const pullRequestSurfaceAvailable = supportsPullRequests && linkedThreadPullRequest !== null; const supportsSettlement = serverConfig?.environment.capabilities.threadSettlement === true; const supportsSnooze = serverConfig?.environment.capabilities.threadSnooze === true; @@ -6813,6 +6828,7 @@ export default function ChatView(props: ChatViewProps) { }, [activeBranchMismatchKey, showBranchMismatchBanner]); const handleSwitchCheckoutToThread = useCallback(async () => { if ( + isCloudThread || !activeProjectCwd || !activeThread || !localCheckoutBranchMismatch || @@ -6867,6 +6883,7 @@ export default function ChatView(props: ChatViewProps) { setIsRestoringThreadBranch(false); scheduleComposerFocus(); }, [ + isCloudThread, activeProjectCwd, activeThread, environmentId, @@ -7721,7 +7738,7 @@ export default function ChatView(props: ChatViewProps) { const onRevertToTurnCount = useCallback( async (turnCount: number, messageId: MessageId, restoreFiles?: boolean) => { const localApi = readLocalApi(); - if (!localApi || !activeThread || isRevertingCheckpoint) return; + if (isCloudThread || !localApi || !activeThread || isRevertingCheckpoint) return; const sourceMessage = serverProjection?.messages.find((message) => message.id === messageId); const message = sourceMessage ? { @@ -7844,6 +7861,7 @@ export default function ChatView(props: ChatViewProps) { } }, [ + isCloudThread, activeThread, activeEnvironmentUnavailable, activeEnvironmentUnavailableLabel, @@ -7866,7 +7884,7 @@ export default function ChatView(props: ChatViewProps) { const onRollbackCheckpoint = useCallback( async (input: { readonly checkpointId: string; readonly scopeId: string }) => { - if (!activeThread || isRevertingCheckpoint) return; + if (isCloudThread || !activeThread || isRevertingCheckpoint) return; if (activeEnvironmentUnavailable && activeEnvironmentUnavailableLabel) { setThreadError( activeThread.id, @@ -7915,6 +7933,7 @@ export default function ChatView(props: ChatViewProps) { [ activeEnvironmentUnavailable, activeEnvironmentUnavailableLabel, + isCloudThread, activeThread, environmentId, isConnecting, @@ -7928,7 +7947,7 @@ export default function ChatView(props: ChatViewProps) { const onForkFromRun = useCallback( async (input: { readonly sourceThreadId: ThreadId; readonly runId: RunId }) => { - if (!activeThread || activeEnvironmentUnavailable) return; + if (isCloudThread || !activeThread || activeEnvironmentUnavailable) return; const targetThreadId = newThreadId(); const targetThreadRef = scopeThreadRef(environmentId, targetThreadId); const result = await forkThreadFromRun({ @@ -7965,6 +7984,7 @@ export default function ChatView(props: ChatViewProps) { }, [ activeEnvironmentUnavailable, + isCloudThread, activeThread, environmentId, forkThreadFromRun, @@ -8700,7 +8720,7 @@ export default function ChatView(props: ChatViewProps) { models: provider.models, modelOptions: selection.options, promptInjectionState: getComposerPromptInjectionState(messageTextForSend), - planModeEnabled: settings.planModeEnabled && !isCloudComposer, + planModeEnabled: settings.planModeEnabled && provider.driverKind !== "kilo-cloud", }); const text = formatOutgoingPrompt({ provider: provider.driverKind, @@ -8718,7 +8738,7 @@ export default function ChatView(props: ChatViewProps) { ), text, interactionMode: resolveComposerInteractionMode({ - planModeEnabled: settings.planModeEnabled && !isCloudComposer, + planModeEnabled: settings.planModeEnabled && provider.driverKind !== "kilo-cloud", provider: provider.snapshot, interactionMode: sendInteractionMode, }).interactionMode, @@ -10738,100 +10758,104 @@ export default function ChatView(props: ChatViewProps) { {/* Messages Wrapper */}
{/* Messages — LegendList handles virtualization and scrolling internally */} - {} : onForkFromRun} - onRollbackCheckpoint={(input) => { - if (!paintOnlyDisplayedTimeline) void onRollbackCheckpoint(input); - }} - supportsConversationRollback={ - !paintOnlyDisplayedTimeline && supportsConversationRollback - } - onRevertToTurnCount={ - paintOnlyDisplayedTimeline ? noopHeldRevert : onRevertTimelineTurn - } - {...(!paintOnlyDisplayedTimeline - ? { onUseArtifactTemplate: useArtifactTemplate } - : {})} - isRevertingCheckpoint={isRevertingCheckpoint} - onImageExpand={onExpandTimelineImage} - onFileOpen={paintOnlyDisplayedTimeline ? noopHeldAttachment : openFileAttachment} - onFileDownload={ - paintOnlyDisplayedTimeline ? noopHeldAttachment : downloadFileAttachment - } - markdownCwd={ - paintOnlyDisplayedTimeline - ? (heldPaintContext?.markdownCwd ?? undefined) - : (gitCwd ?? undefined) - } - resolvedTheme={resolvedTheme} - timestampFormat={timestampFormat} - workspaceRoot={ - paintOnlyDisplayedTimeline - ? (heldPaintContext?.workspaceRoot ?? undefined) - : activeWorkspaceRoot - } - skills={ - activeProviderStatus - ? resolveProviderSkillsForCwd(activeProviderStatus, gitCwd) - : EMPTY_PROVIDER_SKILLS - } - anchorMessageId={paintOnlyDisplayedTimeline ? null : timelineAnchorMessageId} - onAnchorReady={onTimelineAnchorReady} - onAnchorSizeChanged={onTimelineAnchorSizeChanged} - contentInsetEndAdjustment={composerTimelineInset} - liveFollowEnabled={!paintOnlyDisplayedTimeline && timelineLiveFollowEnabled} - onIsAtEndChange={onIsAtEndChange} - onContentOverflowChange={setTimelineOverflows} - onToolOutputCollapsedAtEnd={onToolOutputCollapsedAtEnd} - onManualNavigation={cancelTimelineLiveFollowForUserNavigation} - cancelPositionRestoreRef={cancelPositionRestoreRef} - hideEmptyPlaceholder={isDraftHeroState || threadDetailLoading} - topFadeEnabled={!hasTimelineTopBanner} - {...(paintOnlyDisplayedTimeline || threadHistoryControls === undefined - ? {} - : { historyControls: threadHistoryControls })} - /> + + {} : onForkFromRun} + onRollbackCheckpoint={(input) => { + if (!paintOnlyDisplayedTimeline) void onRollbackCheckpoint(input); + }} + supportsConversationRollback={ + !paintOnlyDisplayedTimeline && supportsConversationRollback + } + onRevertToTurnCount={ + paintOnlyDisplayedTimeline ? noopHeldRevert : onRevertTimelineTurn + } + {...(!paintOnlyDisplayedTimeline + ? { onUseArtifactTemplate: useArtifactTemplate } + : {})} + isRevertingCheckpoint={isRevertingCheckpoint} + onImageExpand={onExpandTimelineImage} + onFileOpen={paintOnlyDisplayedTimeline ? noopHeldAttachment : openFileAttachment} + onFileDownload={ + paintOnlyDisplayedTimeline ? noopHeldAttachment : downloadFileAttachment + } + markdownCwd={ + paintOnlyDisplayedTimeline + ? (heldPaintContext?.markdownCwd ?? undefined) + : (gitCwd ?? undefined) + } + resolvedTheme={resolvedTheme} + timestampFormat={timestampFormat} + workspaceRoot={ + paintOnlyDisplayedTimeline + ? (heldPaintContext?.workspaceRoot ?? undefined) + : activeWorkspaceRoot + } + skills={ + activeProviderStatus + ? resolveProviderSkillsForCwd(activeProviderStatus, gitCwd) + : EMPTY_PROVIDER_SKILLS + } + anchorMessageId={paintOnlyDisplayedTimeline ? null : timelineAnchorMessageId} + onAnchorReady={onTimelineAnchorReady} + onAnchorSizeChanged={onTimelineAnchorSizeChanged} + contentInsetEndAdjustment={composerTimelineInset} + liveFollowEnabled={!paintOnlyDisplayedTimeline && timelineLiveFollowEnabled} + onIsAtEndChange={onIsAtEndChange} + onContentOverflowChange={setTimelineOverflows} + onToolOutputCollapsedAtEnd={onToolOutputCollapsedAtEnd} + onManualNavigation={cancelTimelineLiveFollowForUserNavigation} + cancelPositionRestoreRef={cancelPositionRestoreRef} + hideEmptyPlaceholder={isDraftHeroState || threadDetailLoading} + topFadeEnabled={!hasTimelineTopBanner} + {...(paintOnlyDisplayedTimeline || threadHistoryControls === undefined + ? {} + : { historyControls: threadHistoryControls })} + /> + {/* scroll to end pill — shown when user has scrolled away from the live edge */} {showScrollToBottom && ( @@ -11288,13 +11312,13 @@ export default function ChatView(props: ChatViewProps) { onAddPullRequest={addPullRequestSurface} onAddPullRequests={addPullRequestsSurface} onAddDevice={addDeviceSurface} - browserAvailable={isPreviewSupportedInRuntime()} + browserAvailable={!isCloudThread && isPreviewSupportedInRuntime()} terminalAvailable={activeProject !== null && !isCloudThread} diffAvailable={isServerThread && isGitRepo && !isCloudThread} filesAvailable={activeProject !== null && !isCloudThread} - pullRequestAvailable={pullRequestSurfaceAvailable} - pullRequestsAvailable={pullRequestsSurfaceAvailable} - deviceAvailable={activeThreadRef !== null} + pullRequestAvailable={!isCloudThread && pullRequestSurfaceAvailable} + pullRequestsAvailable={!isCloudThread && pullRequestsSurfaceAvailable} + deviceAvailable={!isCloudThread && activeThreadRef !== null} > {rightPanelContent} @@ -11343,13 +11367,13 @@ export default function ChatView(props: ChatViewProps) { onAddPullRequest={addPullRequestSurface} onAddPullRequests={addPullRequestsSurface} onAddDevice={addDeviceSurface} - browserAvailable={isPreviewSupportedInRuntime()} + browserAvailable={!isCloudThread && isPreviewSupportedInRuntime()} terminalAvailable={activeProject !== null && !isCloudThread} diffAvailable={isServerThread && isGitRepo && !isCloudThread} filesAvailable={activeProject !== null && !isCloudThread} - pullRequestAvailable={pullRequestSurfaceAvailable} - pullRequestsAvailable={pullRequestsSurfaceAvailable} - deviceAvailable={activeThreadRef !== null} + pullRequestAvailable={!isCloudThread && pullRequestSurfaceAvailable} + pullRequestsAvailable={!isCloudThread && pullRequestsSurfaceAvailable} + deviceAvailable={!isCloudThread && activeThreadRef !== null} > {rightPanelContent} diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md index 175f74994c59..f09edf4740ef 100644 --- a/docs/user/providers-kilo.md +++ b/docs/user/providers-kilo.md @@ -1,68 +1,46 @@ # Kilo -The native Kilo provider is a preview. Install Kilo CLI **7.8.3** on the machine -running the T3 environment, then add Kilo in **Settings > Providers**. Set the -binary path if `kilo` is not on that machine's PATH. Other CLI versions are -rejected because their protocol has not been verified with this provider. +Local Kilo execution is disabled in this preview. T3 refuses every local process +start, including provider checks, new prompts, text generation and restored +sessions. CLI 7.8.3 can start MCP commands and connections before approval despite +pure mode, disabled project configuration and deny-all session permissions. Both +legacy project directories and global/profile sources are affected. The Kilo tool +sandbox does not cover MCP startup or reconnect. -Each provider instance has a separate account profile. T3 does not copy credentials -from your ordinary Kilo installation. You can supply an existing `KILO_API_KEY` -in the instance's environment settings or select an existing profile directory. -That directory must contain the `config`, `data`, `cache`, and `state` directories -used as Kilo's XDG roots. It is not the directory containing `auth.json` alone. +No released runtime has a verified fix. Restoring local support requires an +explicitly supported dependency with a process-level MCP policy boundary, tested +across startup, prompts, configuration changes, reconnect and resume. A profile +scan or disabling known server names is insufficient. T3 does not alter your +configuration or patch the installed CLI. There is no unsafe-execution override. +Existing local history and account configuration are retained. The earlier local +prompt, approval, fork, rewind, subagent and text-generation tests are historical +conformance evidence, not currently available functionality. Local Kilo and cloud +parallel execution is consequently blocked. Restricted-mode subagents remain +disallowed even if local execution is restored. -Use a separate instance and profile for each account. Changing the account profile -or instance environment retires its running processes. Existing threads cannot -resume under the replacement account. A Ready status confirms local CLI readiness; -it does not prove that a model account is authenticated or has available credit. - -Local processes use a fixed credential snapshot. Replacing credentials in the same -profile retires its processes and prevents old threads from resuming after reload. -Reload the provider after login or token refresh and start a new thread. Credential -refresh is conservatively treated as an account change because local account -identity cannot be verified without contacting each model provider. - -Repository Kilo configuration and external plugins are disabled. Plugins execute -outside session approval rules, so instance environment settings cannot enable -them. Configure models in the selected account profile or explicit instance -configuration instead. Kilo's built-in authentication plugins remain available. - -**Known security limitation:** CLI 7.8.3 still imports legacy `.kilo/mcp.json` and -`.kilocode/mcp.json` despite disabling project configuration. Their MCP commands -can start before session permission checks. The CLI has no supported blanket MCP -disable, and configuration reloads make a preflight check insufficient. Restricted -execution is therefore not safe for untrusted repositories with these settings. -Disabling external plugins does not resolve this separate limitation. - -Prompts run in the selected T3 workspace. Use separate T3 worktrees for tasks that -must not share files. Separate conversation IDs alone do not isolate a checkout. -Stop terminates the task's owned local process group. A later prompt can restore -its saved conversation in a new process. Rewind copies the retained native -conversation and lets T3 restore the selected file checkpoint. - -Subagents require Full access with no additional approval or sandbox restrictions. -Kilo's own configured child approvals still apply. Restricted modes and Plan mode -block subagent creation because Kilo 7.8.3 does not propagate T3's approval rules -to children. Background subagents and independent child cancellation are not -supported. Tool results appear when the tool finishes; live tool output is not -advertised. - -You can select models and control tasks from web, desktop, and mobile clients -connected to the environment. Configure account profiles in web or desktop -settings. A disconnected client does not stop its task. +| Capability | Local Kilo | Kilo Cloud | +| ------------------------------------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------------------ | +| New prompts and follow-up | Blocked before process start | Full access only | +| Concurrent isolated threads | Blocked | Separate remote worktrees and task identities | +| Local/cloud parallel tasks | Blocked | Requires a safe local runtime | +| History and recovery | Stored history retained; native resume blocked | Durable task identity, paginated admission recovery; no blind resubmit | +| Stop | No local process starts | Requests inference interruption; sleep and compute observed separately | +| Approvals and questions | Blocked | Handles interactions emitted by the remote runtime; cannot enforce restricted policy | +| Rewind, fork, local files, checkpoints, text generation | Blocked | Not supported | +| Subagents | Blocked | Remote Full access may execute them; child history not integrated | +| Web, desktop and mobile | Shows execution-blocked status | Selection, account/model settings and task status; local workspace controls disabled | For remote execution, add a separate **Kilo Cloud** instance in Settings > Providers. Select a profile signed in through the official Kilo login, a GitHub repository that account can access, its branch, and a model. Enable paid cloud execution only when you want prompts and that repository sent to Kilo. T3 never uploads your local -checkout or uncommitted changes. Local Kilo and Kilo Cloud can run concurrently in -separate threads; each cloud thread has its own remote worktree. +checkout or uncommitted changes. Each cloud thread has its own remote worktree. Local Kilo concurrency awaits a +safe native runtime. Cloud execution currently requires **Full access**. The deployed cloud runtime does not apply custom agent permissions, so T3 refuses restricted and Plan modes before submitting a paid task. Shell, edits and subagents cannot be restricted in -cloud Full access. Cloud subagent history is not integrated. Use local Kilo when -you need approvals or restricted execution. Inherited Kilo profiles with setup, +cloud Full access. Cloud subagent history is not integrated. Local Kilo is also unavailable while the runtime safety gate is in place. Inherited Kilo profiles with setup, MCP, skills, agents or environment variables are rejected before a new cloud task. Cloud prompts, native history and follow-up messages use the same remote session. @@ -78,3 +56,11 @@ sleep are separate events. The thread shows task, sandbox and compute status separately. Compute estimates can cover a shared account sandbox and are not a per-task invoice. Unknown or settling status does not mean billing has stopped. T3 does not top up credit or force a sandbox to sleep. + +Admission recovery retains scan progress and follows customer API cursors. An +unresolved start remains uncertain and is not automatically submitted again. +If the remote API reports completion without the corresponding final reply, +T3 keeps the turn unresolved instead of inventing a successful result. This +recovery case still needs a verified terminal contract; inspect the task in Kilo. +A failed or interrupted task can finish even if its history is incomplete. +Reopening the thread can retry history retrieval. From e5c0af359f25131b7f815d9bdda3f2f91a5dbeda Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 18:42:27 +0000 Subject: [PATCH 13/44] test(kilo): use typed event listener in account fixture --- apps/server/src/provider/kilo/KiloCloudAccount.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/provider/kilo/KiloCloudAccount.test.ts b/apps/server/src/provider/kilo/KiloCloudAccount.test.ts index 96dcd897254c..eb718797c119 100644 --- a/apps/server/src/provider/kilo/KiloCloudAccount.test.ts +++ b/apps/server/src/provider/kilo/KiloCloudAccount.test.ts @@ -31,7 +31,7 @@ it.live("separates credential rejection, account support and temporary profile f ); }); server.listen(0, "127.0.0.1"); - await NodeEvents.once(server, "listening"); + await NodeEvents.EventEmitter.once(server, "listening"); return server; }), (server) => From 86b96be09e3d122b2c8bb6b63a1a5912177dc717 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 18:48:39 +0000 Subject: [PATCH 14/44] fix(kilo): refresh mobile workspace gate and verify blocked UI --- apps/desktop/scripts/kilo-ui-evidence.mjs | 2 +- apps/mobile/src/state/use-selected-thread-git-actions.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/desktop/scripts/kilo-ui-evidence.mjs b/apps/desktop/scripts/kilo-ui-evidence.mjs index 9e5882d7cd70..6355db07cd7f 100644 --- a/apps/desktop/scripts/kilo-ui-evidence.mjs +++ b/apps/desktop/scripts/kilo-ui-evidence.mjs @@ -244,7 +244,7 @@ try { await page.getByRole("button", { name: "Add provider", exact: true }).waitFor(); if (verifyBlocked) { await page.getByRole("button", { name: "Select Kilo", exact: true }).click(); - await page.getByText("Local Kilo execution is disabled:", { exact: false }).waitFor(); + await page.getByText("Local Kilo execution is disabled:", { exact: false }).first().waitFor(); await page.screenshot({ animations: "disabled", path: NodePath.join(evidence, "local-execution-blocked.png"), diff --git a/apps/mobile/src/state/use-selected-thread-git-actions.ts b/apps/mobile/src/state/use-selected-thread-git-actions.ts index a0ee160c9add..305147199c98 100644 --- a/apps/mobile/src/state/use-selected-thread-git-actions.ts +++ b/apps/mobile/src/state/use-selected-thread-git-actions.ts @@ -120,7 +120,7 @@ export function useSelectedThreadGitActions() { return; } void refreshSelectedThreadGitStatus({ quiet: true }); - }, [refreshSelectedThreadGitStatus, selectedThread, selectedThreadProject]); + }, [refreshSelectedThreadGitStatus, selectedThread, selectedThreadCwd, selectedThreadProject]); const runSelectedThreadGitMutation = useCallback( async ( From 74d0a46494ba89b7e193c442e747218cab22eb91 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 19:57:48 +0000 Subject: [PATCH 15/44] fix(kilo): trust native configuration and recover cloud results --- apps/desktop/scripts/kilo-ui-evidence.mjs | 91 ++-- .../SettingsProviderAccountsRouteScreen.tsx | 9 +- .../features/threads/ThreadDetailScreen.tsx | 5 +- .../Adapters/KiloAdapterV2.live.test.ts | 23 +- .../Adapters/KiloAdapterV2.ts | 2 +- .../Adapters/KiloCloudAdapterV2.test.ts | 452 +++++++++++++++++- .../Adapters/KiloCloudAdapterV2.ts | 287 ++++++++--- .../server/src/provider/Drivers/KiloDriver.ts | 13 +- .../src/provider/kilo/KiloCloudJournal.ts | 28 +- .../kilo/KiloRuntime.crash.fixture.mjs | 43 ++ .../provider/kilo/KiloRuntime.live.test.ts | 156 +++++- .../provider/kilo/KiloRuntime.safety.test.ts | 99 ---- apps/server/src/provider/kilo/KiloRuntime.ts | 43 +- apps/web/src/components/ChatView.tsx | 5 +- docs/user/providers-kilo.md | 147 +++--- packages/contracts/src/orchestrationV2.ts | 3 + packages/contracts/src/settings.ts | 3 +- 17 files changed, 1068 insertions(+), 341 deletions(-) create mode 100644 apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs delete mode 100644 apps/server/src/provider/kilo/KiloRuntime.safety.test.ts diff --git a/apps/desktop/scripts/kilo-ui-evidence.mjs b/apps/desktop/scripts/kilo-ui-evidence.mjs index 6355db07cd7f..ca626241b9ce 100644 --- a/apps/desktop/scripts/kilo-ui-evidence.mjs +++ b/apps/desktop/scripts/kilo-ui-evidence.mjs @@ -10,9 +10,6 @@ import * as NodeUtil from "node:util"; import { chromium } from "playwright-core"; if (!process.env.KILO_BIN) throw new Error("KILO_BIN must point to the pinned local CLI"); -const verifyBlocked = process.env.KILO_VERIFY_BLOCKED === "1"; -if (verifyBlocked && process.env.KILO_CLOUD_TEST_PROFILE) - throw new Error("Blocked-runtime verification must use no live cloud profile"); const root = NodePath.resolve(import.meta.dirname, "../../.."); const temporary = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-kilo-ui-")); const evidence = process.env.KILO_EVIDENCE_DIR ?? NodePath.join(temporary, "evidence"); @@ -137,6 +134,7 @@ await NodeFSP.writeFile( enabled: true, config: { binaryPath: process.env.KILO_BIN, accountId: "ui-fixture" }, environment: [ + { name: "HOME", value: temporary }, { name: "KILO_CONFIG_CONTENT", value: JSON.stringify(config) }, ...[ "KILO_DISABLE_MODELS_FETCH", @@ -198,58 +196,57 @@ try { await page.getByText("Local folder", { exact: true }).click(); await page.getByPlaceholder("Enter path (e.g. ~/projects/my-app)").fill(workspace); await page.getByPlaceholder("Enter path (e.g. ~/projects/my-app)").press("Enter"); - if (!verifyBlocked) { - if (process.env.KILO_CLOUD_TEST_PROFILE) { - await page.locator("[data-chat-provider-model-picker-label]").click(); - await page.getByPlaceholder("Search models...").fill("deepseek-v4.1-flash"); - await page.getByText("deepseek/deepseek-v4.1-flash", { exact: true }).last().click(); - await page - .getByText("Closing T3 does not stop remote work or billing.", { exact: false }) - .waitFor(); - await page.getByRole("button", { name: "Unknown", exact: true }).click(); - await page.getByRole("menuitemradio", { name: /^Low/ }).click(); - await page.getByRole("button", { name: "Low", exact: true }).waitFor(); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "cloud-before-send.png"), - }); - } + if (process.env.KILO_CLOUD_TEST_PROFILE) { await page.locator("[data-chat-provider-model-picker-label]").click(); - await page.getByPlaceholder("Search models...").fill("Local fixture"); - await page.getByText("Local fixture", { exact: true }).last().click(); - await page.getByRole("button", { name: "Local fixture", exact: true }).waitFor(); - await page.locator("[contenteditable=true]").fill("Kilo local integration: say hello."); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "before-send.png"), - }); - await page.getByRole("button", { name: "Submit message", exact: true }).click(); - await page.getByText(answer, { exact: true }).waitFor({ timeout: 60000 }); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "streamed-answer.png"), - }); - await page.getByRole("button", { name: "Submit message", exact: true }).waitFor(); + await page.getByPlaceholder("Search models...").fill("deepseek-v4.1-flash"); + await page.getByText("deepseek/deepseek-v4.1-flash", { exact: true }).last().click(); + await page + .getByText("Closing T3 does not stop remote work or billing.", { exact: false }) + .waitFor(); + await page.getByRole("button", { name: "Unknown", exact: true }).click(); + await page.getByRole("menuitemradio", { name: /^Low/ }).click(); + await page.getByRole("button", { name: "Low", exact: true }).waitFor(); await page.screenshot({ animations: "disabled", - path: NodePath.join(evidence, "completed-answer.png"), + path: NodePath.join(evidence, "cloud-before-send.png"), }); - console.log("Local native answer rendered; opening provider settings."); } + await page.locator("[data-chat-provider-model-picker-label]").click(); + await page.getByPlaceholder("Search models...").fill("Local fixture"); + await page.getByText("Local fixture", { exact: true }).last().click(); + await page.getByRole("button", { name: "Local fixture", exact: true }).waitFor(); + await page.locator("[contenteditable=true]").fill("Kilo local integration: say hello."); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "before-send.png"), + }); + await page.getByRole("button", { name: "Submit message", exact: true }).click(); + await page.getByText(answer, { exact: true }).waitFor({ timeout: 60000 }); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "streamed-answer.png"), + }); + await page.getByRole("button", { name: "Submit message", exact: true }).waitFor(); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "completed-answer.png"), + }); + console.log("Local native answer rendered; opening provider settings."); await page.getByRole("button", { name: "Settings", exact: true }).click(); await page.waitForURL("**/settings/general*"); await page.getByText("Restore device defaults", { exact: true }).waitFor(); await page.getByRole("button", { name: "Providers", exact: true }).click(); await page.waitForURL("**/settings/providers*"); await page.getByRole("button", { name: "Add provider", exact: true }).waitFor(); - if (verifyBlocked) { - await page.getByRole("button", { name: "Select Kilo", exact: true }).click(); - await page.getByText("Local Kilo execution is disabled:", { exact: false }).first().waitFor(); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "local-execution-blocked.png"), - }); - } + await page.getByRole("button", { name: "Select Kilo", exact: true }).click(); + await page + .getByText("Native configuration and MCP servers are trusted", { exact: false }) + .first() + .waitFor(); + await page.screenshot({ + animations: "disabled", + path: NodePath.join(evidence, "local-trust-settings.png"), + }); await page.getByRole("button", { name: "Select Kilo Cloud", exact: true }).click(); if (process.env.KILO_CLOUD_TEST_PROFILE) { const consent = page.getByRole("switch", { name: "Allow paid cloud execution", exact: true }); @@ -273,16 +270,14 @@ try { path: NodePath.join(evidence, "provider-settings.png"), }); await context.close(); - if (verifyBlocked && requests !== 0) throw new Error("Blocked execution reached inference"); - if (!verifyBlocked && !requests) - throw new Error("The real CLI did not contact the local inference fixture"); + if (!requests) throw new Error("The real CLI did not contact the local inference fixture"); await NodeFSP.writeFile( NodePath.join(evidence, "verification.json"), JSON.stringify( { commit: (await execFile("git", ["rev-parse", "HEAD"], { cwd: root })).stdout.trim(), inferenceRequests: requests, - localExecutionBlocked: verifyBlocked, + nativeConfigurationTrusted: true, inference: "loopback fixture only", client: "Chromium web", }, diff --git a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx index 07d1cd3c6677..a07c80b4067f 100644 --- a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx @@ -51,7 +51,10 @@ export function SettingsProviderAccountsRouteScreen() { ) .map((provider) => provider.driver === "kilo" || provider.driver === "kilo-cloud" ? ( - + {provider.displayName} @@ -62,7 +65,7 @@ export function SettingsProviderAccountsRouteScreen() { {provider.driver === "kilo-cloud" ? "Kilo Cloud runs in a remote repository and uses Kilo credit. Closing T3 does not stop remote work or billing. Manage its account and repository in web or desktop Settings." - : "Kilo runs on this environment. Manage its isolated account profile in web or desktop Settings."} + : "Kilo runs on this environment and trusts native configuration, plugins and MCP servers. Tool approvals are not a sandbox. Manage its account profile in web or desktop Settings."} ) : ( @@ -196,7 +199,7 @@ function ProviderAccount({ } return ( - + {provider.displayName ?? provider.driver} diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index a0593c69c1d2..9b8543a2623f 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -1276,8 +1276,9 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread Last observation: {cloudExecution.observedAt ?? "unavailable"}. Task:{" "} - {cloudExecution.task}. Sandbox: {cloudExecution.sandbox}. Compute:{" "} - {cloudExecution.billing} + {cloudExecution.task}.{" "} + {cloudExecution.result ? `Result: ${cloudExecution.result}. ` : ""}Sandbox:{" "} + {cloudExecution.sandbox}. Compute: {cloudExecution.billing} {cloudExecution.billingAttribution === "payer_shared" ? " (shared account)" : ""} diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts index e8d9e82cd800..c24883f0292a 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts @@ -244,8 +244,7 @@ const inference = Effect.acquireRelease( }), ); -// Historical native conformance coverage. Re-enable only after an audited MCP runtime fix. -describe.skip("Kilo adapter with native runtime and local inference", () => { +describe.skipIf(!binary)("Kilo adapter with native runtime and local inference", () => { it.live( "delivers a real streamed turn and restores its native history", () => @@ -253,6 +252,23 @@ describe.skip("Kilo adapter with native runtime and local inference", () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-adapter-" }); + // Both legacy MCP sources are trusted native config, even when project + // discovery is disabled. The harmless fixture proves actual execution. + for (const dir of [".kilo", ".kilocode"]) { + yield* fs.makeDirectory(path.join(root, dir)); + const program = path.join(root, `${dir}-mcp.cjs`); + yield* fs.writeFileString( + program, + `require('node:fs').writeFileSync(${encodeJson(path.join(root, `${dir}-marker`))}, String(process.pid)); +require('node:readline').createInterface({input:process.stdin}).on('line',line=>{const m=JSON.parse(line);if(m.id!==undefined)process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:m.id,result:m.method==='initialize'?{protocolVersion:'2024-11-05',capabilities:{tools:{}},serverInfo:{name:'fixture',version:'1'}}:{tools:[]}})+'\\n')});`, + ); + yield* fs.writeFileString( + path.join(root, dir, "mcp.json"), + encodeJson({ + mcpServers: { [dir.slice(1)]: { command: process.execPath, args: [program] } }, + }), + ); + } const model = yield* inference; const continuationKey = "account-scope"; const instanceId = ProviderInstanceId.make("kilo-test"); @@ -269,6 +285,7 @@ describe.skip("Kilo adapter with native runtime and local inference", () => { profileDirectory: path.join(root, "profile"), environment: { PATH: process.env.PATH, + HOME: root, HTTP_PROXY: process.env.HTTP_PROXY, HTTPS_PROXY: process.env.HTTPS_PROXY, NO_PROXY: process.env.NO_PROXY, @@ -422,6 +439,8 @@ describe.skip("Kilo adapter with native runtime and local inference", () => { "Hello from local Kilo.", ); assert.isAbove(model.requests.length, 0); + for (const dir of [".kilo", ".kilocode"]) + assert.isTrue(yield* fs.exists(path.join(root, `${dir}-marker`))); assert.isTrue( seen.some((e) => e.type === "turn_item.updated" && e.turnItem.type === "reasoning"), ); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts index f5fb9a95fce5..19e2f6e58bb1 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts @@ -160,7 +160,7 @@ const wire = (effect: Effect.Effect) => ), ); -// Kilo 7.8.3 task.ts persists inherited edit/bash/MCP ceilings before launching a child. +// Native tool rules are approval policy, not process or MCP-start isolation. const permissions = (policy: Adapter.ProviderAdapterV2RuntimePolicy) => { const rules = openCodePermissionRules(policy); // Kilo re-appends session denies in Plan and inherits them into children. OpenCode's diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index de16417cd367..6557e49e80bf 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -1,4 +1,7 @@ // @effect-diagnostics nodeBuiltinImport:off - external customer API contract over a loopback socket. +import * as NodeChildProcess from "node:child_process"; +import * as KiloRuntime from "../../provider/kilo/KiloRuntime.ts"; +import * as KiloAdapter from "./KiloAdapterV2.ts"; import * as NodeHttp from "node:http"; import * as NodeEvents from "node:events"; import * as NodeServices from "@effect/platform-node/NodeServices"; @@ -16,6 +19,8 @@ import { } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Exit from "effect/Exit"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -37,16 +42,36 @@ const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); const fixture = Effect.acquireRelease( Effect.promise(async () => { let submissions = 0; + let localResponse: NodeHttp.ServerResponse | undefined; + let localRequests = 0; + let overlapped = false; + const completeLocal = () => { + if (!localResponse || submissions < 2) return; + overlapped = true; + localResponse.writeHead(200, { "content-type": "text/event-stream" }); + for (const choice of [ + { delta: { role: "assistant", content: "Local parallel reply" }, finish_reason: null }, + { delta: {}, finish_reason: "stop" }, + ]) + localResponse.write( + `data: ${JSON.stringify({ id: "chatcmpl-parallel", object: "chat.completion.chunk", created: 1, model: "test", choices: [{ index: 0, ...choice }] })}\n\n`, + ); + localResponse.end("data: [DONE]\n\n"); + localResponse = undefined; + }; let signalInterrupt!: () => void; const interruptSeen = new Promise((resolve) => { signalInterrupt = resolve; }); const control = { status: "completed", + requireLocalOverlap: false, dropNextPrepare: false, omittedItemCount: 0, missingHistory: false, incompleteHistory: false, + historyMode: "normal", + historyReads: 0, interruptAccepted: false, interruptPosts: 0, permission: false, @@ -73,6 +98,12 @@ const fixture = Effect.acquireRelease( }); request.on("end", () => { const url = new URL(request.url!, "http://localhost"); + if (url.pathname.endsWith("/chat/completions")) { + localRequests++; + localResponse = response; + completeLocal(); + return; + } const operation = url.pathname.split("/").at(-1)!; const input = JSON.parse(raw || url.searchParams.get("input") || "{}") as Record< string, @@ -85,6 +116,7 @@ const fixture = Effect.acquireRelease( if (operation.startsWith("agentProfiles.")) return reply([]); if (operation === "cloudAgentNext.prepareSession") { submissions++; + completeLocal(); const suffix = String(submissions).padStart(12, "0"); const state = { cloud: `workspace_12345678-1234-1234-1234-${suffix}`, @@ -207,10 +239,84 @@ const fixture = Effect.acquireRelease( return reply({ cloudAgentSessionId: state.cloud, messageId: input.messageId, - status: control.status, + status: control.requireLocalOverlap && !localRequests ? "running" : control.status, }); if (operation === "cliSessionsV2.getSessionMessagesPage" && control.missingHistory) return reply({ kiloSessionId: state.native, history: null, watermarkEventId: 49 }); + if ( + operation === "cliSessionsV2.getSessionMessagesPage" && + control.historyMode !== "normal" + ) { + control.historyReads++; + const message = + control.historyMode === "older" && input.cursor === "1" + ? state.messages[0]! + : state.messages.at(-1)!; + const part = { + id: `tool-${message.id}`, + sessionID: state.native, + messageID: `reply-${message.id}`, + type: "tool", + tool: "read", + callID: `call-${message.id}`, + state: { status: "completed", input: {}, output: "synthetic" }, + }; + const assistant = { + info: { + id: `reply-${message.id}`, + sessionID: state.native, + role: "assistant", + parentID: message.id, + time: { created: 1, completed: 2 }, + }, + parts: control.historyMode === "empty" ? [] : [part], + }; + const user = { + info: { id: message.id, sessionID: state.native, role: "user", time: { created: 1 } }, + parts: [], + }; + const page = Number(input.cursor ?? 0); + const more = control.historyMode === "paged" && page < 4; + return reply({ + kiloSessionId: state.native, + watermarkEventId: 3, + history: { + nextCursor: + control.historyMode === "repeated" || + (control.historyMode === "older" && !input.cursor) + ? "1" + : more + ? String(page + 1) + : null, + omittedItemCount: 0, + messages: more + ? [] + : [ + assistant, + ...(control.historyMode === "unfinished" + ? [ + { + info: { + ...assistant.info, + id: `newer-${message.id}`, + time: { created: 3 }, + }, + parts: [ + { + ...part, + id: `newer-tool-${message.id}`, + messageID: `newer-${message.id}`, + state: { status: "running", input: {} }, + }, + ], + }, + ] + : []), + ...(control.historyMode === "repeated" ? [] : [user]), + ], + }, + }); + } if (operation === "cliSessionsV2.getSessionMessagesPage") return reply({ kiloSessionId: state.native, @@ -272,6 +378,8 @@ const fixture = Effect.acquireRelease( return { origin: `http://127.0.0.1:${address.port}`, submissions: () => submissions, + localRequests: () => localRequests, + overlapped: () => overlapped, conversations, control, interruptSeen, @@ -310,6 +418,54 @@ it.live( journal, }; const adapter = yield* CloudAdapter.make(adapterOptions); + const localInstance = ProviderInstanceId.make("native-parallel"); + const localSelection = { instanceId: localInstance, model: "fixture/test" }; + const localCwd = `${directory}/local`; + let localAdapter: Adapter.ProviderAdapterV2Shape | undefined; + if (process.env.KILO_BIN) { + remote.control.requireLocalOverlap = true; + yield* fs.makeDirectory(localCwd); + yield* Effect.promise( + () => + new Promise((resolve, reject) => + NodeChildProcess.execFile("git", ["init", "--quiet", localCwd], (error) => + error ? reject(error) : resolve(), + ), + ), + ); + const native = yield* KiloRuntime.make({ + instanceId: "native-parallel", + binaryPath: process.env.KILO_BIN, + profileDirectory: `${directory}/native-profile`, + environment: { + PATH: process.env.PATH, + HOME: directory, + KILO_DISABLE_MODELS_FETCH: "1", + KILO_DISABLE_DEFAULT_PLUGINS: "1", + KILO_DISABLE_EXTERNAL_SKILLS: "1", + KILO_CONFIG_CONTENT: yield* encodeJson({ + model: "fixture/test", + small_model: "fixture/test", + plugin: [], + provider: { + fixture: { + npm: "@ai-sdk/openai-compatible", + name: "Loopback", + options: { baseURL: `${remote.origin}/v1` }, + models: { test: { name: "Test", limit: { context: 10000, output: 1000 } } }, + }, + }, + }), + }, + }); + localAdapter = yield* KiloAdapter.make({ + instanceId: localInstance, + continuationKey: "native-parallel", + cwd: localCwd, + attachmentsDir: `${directory}/attachments`, + runtime: native, + }); + } yield* Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; const done = yield* Deferred.make(); @@ -321,13 +477,14 @@ it.live( ["completed", "failed", "interrupted"].includes(event.payload.status) ) completed.add(event.threadId); - return completed.size === 2 + return completed.size === (localAdapter ? 3 : 2) ? Deferred.succeed(done, undefined).pipe(Effect.asVoid) : Effect.void; }), Effect.forkScoped, ); - for (const suffix of ["a", "b"]) { + for (const suffix of [...(localAdapter ? ["local"] : []), "a", "b"]) { + const selectedModel = suffix === "local" ? localSelection : modelSelection; const threadId = ThreadId.make(`cloud-${suffix}`); yield* orchestrator.dispatch({ type: "thread.create", @@ -337,11 +494,11 @@ it.live( threadId, projectId: ProjectId.make("cloud-project"), title: "Cloud contract", - modelSelection, + modelSelection: selectedModel, runtimeMode: "full-access", interactionMode: "default", branch: null, - worktreePath: `${directory}/must-not-exist-${suffix}`, + worktreePath: suffix === "local" ? localCwd : `${directory}/must-not-exist-${suffix}`, }); yield* orchestrator.dispatch({ type: "message.dispatch", @@ -352,12 +509,28 @@ it.live( messageId: MessageId.make(`user-${suffix}`), text: `isolation-${suffix}`, attachments: [], - modelSelection, + modelSelection: selectedModel, dispatchMode: { type: "start_immediately" }, }); } yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); yield* Deferred.await(done); + if (localAdapter) { + assert.isTrue(remote.overlapped()); + assert.isAbove(remote.localRequests(), 0); + const projection = yield* orchestrator.getThreadProjection(ThreadId.make("cloud-local")); + assert.equal( + projection.runs[0]?.status, + "completed", + yield* encodeJson(projection.turnItems), + ); + assert.isTrue( + projection.messages.some((message) => message.text === "Local parallel reply"), + ); + assert.isFalse( + projection.messages.some((message) => message.text.includes("Remote reply")), + ); + } for (const suffix of ["a", "b"]) { const projection = yield* orchestrator.getThreadProjection( ThreadId.make(`cloud-${suffix}`), @@ -417,25 +590,28 @@ it.live( Effect.provide( makeOrchestratorV2ReplayLayerWithRegistry( { name: "kilo-cloud-contract" }, - Registry.makeSingleLayer({ - ...adapter, - openSession: (input) => - adapter.openSession(input).pipe( - Effect.map((runtime) => ({ - ...runtime, - startTurn: (input) => - runtime - .startTurn(input) - .pipe( - Effect.catchCause((cause) => - Effect.logError(Cause.pretty(cause)).pipe( - Effect.andThen(Effect.failCause(cause)), + Registry.makeLayer([ + { + ...adapter, + openSession: (input) => + adapter.openSession(input).pipe( + Effect.map((runtime) => ({ + ...runtime, + startTurn: (input) => + runtime + .startTurn(input) + .pipe( + Effect.catchCause((cause) => + Effect.logError(Cause.pretty(cause)).pipe( + Effect.andThen(Effect.failCause(cause)), + ), ), ), - ), - })), - ), - }), + })), + ), + }, + ...(localAdapter ? [localAdapter] : []), + ]), ), ), ); @@ -655,6 +831,234 @@ it.live( assert.isFalse(yield* restored.hasPendingBackgroundWork!); assert.equal((yield* journal.read).at(-1)?.state, "failed"); assert.equal(remote.submissions(), 2); + // A completed workspace can have no ingested output. Persist the retrieval + // deadline, restart the adapter, and fail locally without changing remote state. + remote.control.status = "completed"; + remote.control.missingHistory = true; + remote.control.omittedItemCount = 0; + remote.control.incompleteHistory = false; + const retrievalScope = yield* Scope.fork(yield* Effect.scope); + const retrieving = yield* adapter + .openSession({ + threadId: restore.threadId, + providerSessionId: ProviderSessionId.make("retrieval"), + modelSelection, + runtimePolicy: restore.runtimePolicy, + }) + .pipe(Effect.provideService(Scope.Scope, retrievalScope)); + const retrievalThread = yield* retrieving.resumeThread({ providerThread: restoredThread }); + const awaiting = yield* Deferred.make(); + yield* retrieving.events.pipe( + Stream.runForEach((event) => + event.type === "provider_session.updated" && + event.providerSession.lastError?.includes("Awaiting its correlated result") + ? Deferred.succeed(awaiting, undefined) + : Effect.void, + ), + Effect.forkIn(retrievalScope), + ); + const retrievalInput = { + ...restore, + reattach: false, + providerThread: retrievalThread, + runId: RunId.make("retrieval-run"), + attemptId: RunAttemptId.make("retrieval-attempt"), + runOrdinal: 4, + providerTurnOrdinal: 4, + message: { + ...restore.message, + messageId: MessageId.make("retrieval"), + text: "Late result", + }, + }; + yield* retrieving.startTurn(retrievalInput); + yield* Deferred.await(awaiting); + const waiting = (yield* journal.read).at(-1)!; + assert.equal(waiting.state, "awaiting_result"); + assert.equal(waiting.remoteState, "completed"); + assert.isDefined(waiting.resultRecovery?.deadlineMs); + assert.isFalse(yield* journal.reserve({ ...waiting, operationKey: "duplicate-retrieval" })); + yield* Scope.close(retrievalScope, Exit.void); + // Simulate reopening after the durable deadline, without wall-clock sleeps. + yield* journal.save({ + ...waiting, + resultRecovery: { ...waiting.resultRecovery!, deadlineMs: 0, nextAttemptMs: 0 }, + }); + const afterRestart = yield* adapter.openSession({ + threadId: restore.threadId, + providerSessionId: ProviderSessionId.make("retrieval-restart"), + modelSelection, + runtimePolicy: restore.runtimePolicy, + }); + const afterThread = yield* afterRestart.resumeThread({ providerThread: retrievalThread }); + const retrievalEvents: Adapter.ProviderAdapterV2Event[] = []; + const retrievalFailed = yield* Deferred.make(); + yield* afterRestart.events.pipe( + Stream.runForEach((event) => { + retrievalEvents.push(event); + return event.type === "turn.terminal" + ? Deferred.succeed(retrievalFailed, undefined) + : Effect.void; + }), + Effect.forkScoped, + ); + remote.control.question = true; + const outstanding = yield* afterRestart.readThreadSnapshot({ providerThread: afterThread }); + assert.isTrue(outstanding.runtimeRequests.some((request) => request.status === "pending")); + assert.equal((yield* journal.read).at(-1)?.state, "awaiting_result"); + // Another client resolves the question; missing output still has a bounded window. + remote.control.question = false; + const afterQuestion = (yield* journal.read).at(-1)!; + yield* journal.save({ + ...afterQuestion, + resultRecovery: { ...afterQuestion.resultRecovery!, deadlineMs: 0, nextAttemptMs: 0 }, + }); + const unavailable = yield* afterRestart.readThreadSnapshot({ providerThread: afterThread }); + yield* Deferred.await(retrievalFailed); + assert.equal(unavailable.providerThread.nativeMetadata?.cloudExecution?.task, "completed"); + assert.equal( + unavailable.providerThread.nativeMetadata?.cloudExecution?.result, + "unavailable", + ); + assert.equal((yield* journal.read).at(-1)?.state, "failed"); + const failure = retrievalEvents.find((event) => event.type === "turn.terminal"); + assert.isTrue( + failure?.type === "turn.terminal" && + failure.status === "failed" && + failure.failure?.message.includes("result could not be retrieved"), + ); + remote.control.missingHistory = false; + const late = yield* afterRestart.readThreadSnapshot({ providerThread: afterThread }); + const repeatedLate = yield* afterRestart.readThreadSnapshot({ providerThread: afterThread }); + assert.equal( + late.messages.filter((message) => message.text === "Remote reply: Late result").length, + 1, + ); + assert.deepEqual( + repeatedLate.messages.map((message) => message.id), + late.messages.map((message) => message.id), + ); + assert.equal((yield* journal.read).at(-1)?.resultStatus, "available"); + assert.equal(retrievalEvents.filter((event) => event.type === "turn.terminal").length, 1); + assert.equal(remote.submissions(), 2); + // Stop during retrieval is local cancellation, never a remote interrupt. + remote.control.missingHistory = true; + const cancelling = yield* adapter.openSession({ + threadId: restore.threadId, + providerSessionId: ProviderSessionId.make("retrieval-cancel"), + modelSelection, + runtimePolicy: restore.runtimePolicy, + }); + const cancelThread = yield* cancelling.resumeThread({ providerThread: afterThread }); + const cancelAwaiting = yield* Deferred.make(); + yield* cancelling.events.pipe( + Stream.runForEach((event) => + event.type === "provider_session.updated" && + event.providerSession.lastError?.includes("Awaiting its correlated result") + ? Deferred.succeed(cancelAwaiting, undefined) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* cancelling.startTurn({ + ...retrievalInput, + providerThread: cancelThread, + runId: RunId.make("cancel-run"), + attemptId: RunAttemptId.make("cancel-attempt"), + runOrdinal: 5, + providerTurnOrdinal: 5, + message: { + ...restore.message, + messageId: MessageId.make("cancel-retrieval"), + text: "Cancel retrieval", + }, + }); + yield* Deferred.await(cancelAwaiting); + const cancellingIntent = (yield* journal.read).at(-1)!; + const interruptPosts = remote.control.interruptPosts; + yield* cancelling.interruptTurn({ + providerThread: cancelThread, + providerTurnId: cancellingIntent.providerTurn.id, + }); + assert.equal(remote.control.interruptPosts, interruptPosts); + const cancelled = (yield* journal.read).at(-1)!; + assert.equal(cancelled.state, "interrupted"); + assert.equal(cancelled.remoteState, "completed"); + assert.equal(cancelled.resultStatus, "cancelled"); + remote.control.missingHistory = false; + for (const [index, mode] of [ + "empty", + "tool-only", + "unfinished", + "repeated", + "paged", + ].entries()) { + remote.control.historyMode = mode; + remote.control.historyReads = 0; + const testScope = yield* Scope.fork(yield* Effect.scope); + const runtime = yield* adapter + .openSession({ + threadId: restore.threadId, + providerSessionId: ProviderSessionId.make(`result-${mode}`), + modelSelection, + runtimePolicy: restore.runtimePolicy, + }) + .pipe(Effect.provideService(Scope.Scope, testScope)); + const selected = yield* runtime.resumeThread({ providerThread: cancelThread }); + const done = yield* Deferred.make(); + const waitingResult = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.runForEach((event) => + Effect.gen(function* () { + if (event.type === "turn.terminal") yield* Deferred.succeed(done, event); + if ( + event.type === "provider_session.updated" && + event.providerSession.lastError?.includes("Awaiting its correlated result") + ) + yield* Deferred.succeed(waitingResult, undefined); + if ( + event.type === "provider_thread.updated" && + event.providerThread.nativeMetadata?.cloudExecution?.task === "admission_unknown" + ) + assert.isUndefined(event.providerThread.nativeMetadata.cloudExecution.result); + }), + ), + Effect.forkIn(testScope), + ); + yield* runtime.startTurn({ + ...retrievalInput, + providerThread: selected, + runId: RunId.make(`result-${mode}`), + attemptId: RunAttemptId.make(`result-attempt-${mode}`), + runOrdinal: 6 + index, + providerTurnOrdinal: 6 + index, + message: { ...restore.message, messageId: MessageId.make(`result-${mode}`), text: mode }, + }); + if (mode === "unfinished" || mode === "repeated") { + yield* Deferred.await(waitingResult); + const unfinished = (yield* journal.read).at(-1)!; + assert.equal(unfinished.state, "awaiting_result"); + yield* runtime.interruptTurn({ + providerThread: selected, + providerTurnId: unfinished.providerTurn.id, + }); + } else { + if (mode === "paged") { + yield* Deferred.await(waitingResult); + assert.equal(remote.control.historyReads, 4); + assert.equal((yield* journal.read).at(-1)?.resultRecovery?.cursor, "4"); + } + const terminal = yield* Deferred.await(done); + assert.isTrue(terminal.type === "turn.terminal" && terminal.status === "completed"); + assert.equal((yield* journal.read).at(-1)?.resultStatus, "available"); + } + yield* Scope.close(testScope, Exit.void); + } + remote.control.historyMode = "older"; + const older = yield* cancelling.readThreadSnapshot({ providerThread: cancelThread }); + assert.isTrue(older.messages.some((message) => message.id === "user-a")); + assert.isTrue(older.messages.some((message) => message.id === "result-paged")); + remote.control.historyMode = "normal"; const first = (yield* journal.read)[0]!; yield* journal.save({ ...first, interruptRequested: true }); assert.equal((yield* replacementForStale()).operation, "write"); @@ -694,5 +1098,5 @@ it.live( 1, ); }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), - 30_000, + 60_000, ); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts index 36d8c23c5e3c..6cb4489f5ece 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -157,6 +157,8 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { readonly client: ReturnType; readonly journal: Effect.Success>; readonly allowAdmission?: boolean; + /** Bounded local retrieval window, persisted from first remote completion. */ + readonly resultRecoveryTimeoutMs?: number; }) { const ids = yield* IdAllocator.IdAllocatorV2; const crypto = yield* Crypto.Crypto; @@ -204,6 +206,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { let admissionProbeDelay = 2_000; let streamCursor = 0; let monitorSandbox = false; + let resultStatus: Journal.CloudIntent["resultStatus"]; let taskState: | "not_started" | "admission_unknown" @@ -280,6 +283,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { }); const finish = Effect.fn("KiloCloudAdapterV2.finish")(function* ( terminal: "completed" | "failed" | "interrupted", + resultFailure?: string, ) { if (!active) return; const at = yield* DateTime.now; @@ -295,7 +299,11 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { status: "idle", nativeMetadata: { ...thread.nativeMetadata, - cloudExecution: { ...thread.nativeMetadata.cloudExecution, task: terminal }, + cloudExecution: { + ...thread.nativeMetadata.cloudExecution, + task: saved.remoteState ?? terminal, + ...(saved.resultStatus ? { result: saved.resultStatus } : {}), + }, }, }; yield* emit({ type: "provider_thread.updated", driver, providerThread: thread }); @@ -348,7 +356,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { failure: makeProviderFailure({ class: "provider_error", code: "provider_error", - message: "Kilo Cloud reported a failed task.", + message: resultFailure ?? "Kilo Cloud reported a failed task.", }), threadDisposition: "reusable", } @@ -373,7 +381,8 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { // the thread can retry history independently of the ended turn. needsHistoryRestore = false; if (!binding) monitorSandbox = false; - taskState = terminal; + taskState = saved.remoteState ?? terminal; + resultStatus = saved.resultStatus; yield* Deferred.succeed(terminalSignal, undefined); }); const project = Effect.fn("KiloCloudAdapterV2.project")(function* ( @@ -647,78 +656,201 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { persisted.revision > active.revision ) active = persisted; - const outcome = active - ? yield* wire(options.client.result(binding, active.messageId)) - : null; + const nowMs = yield* Clock.currentTimeMillis; + const previousRecovery = active?.resultRecovery; + if (previousRecovery && nowMs < previousRecovery.nextAttemptMs) return; + const outcome = + active?.remoteState === "completed" + ? { status: "completed" as const } + : active + ? yield* wire(options.client.result(binding, active.messageId)) + : null; if (active?.messageId !== expectedMessageId) return; - if (outcome) taskState = outcome.status; + if (outcome && active) { + taskState = outcome.status; + // First observed completion and its deadline are one durable write. + if (outcome.status === "completed" && !active.resultRecovery) { + yield* save({ + ...active, + remoteState: "completed", + state: "awaiting_result", + resultStatus: "awaiting_result", + resultRecovery: { + deadlineMs: nowMs + (options.resultRecoveryTimeoutMs ?? 300_000), + nextAttemptMs: nowMs, + attempts: 0, + cursor: null, + seenCursors: [], + }, + }); + resultStatus = "awaiting_result"; + } else if (active.remoteState !== outcome.status) + yield* save({ ...active, remoteState: outcome.status }); + } + // Reserve the next attempt before I/O, including pending/history failures. + if (active?.resultRecovery) { + const recovery = active.resultRecovery; + yield* save({ + ...active, + resultRecovery: { + ...recovery, + attempts: recovery.attempts + 1, + nextAttemptMs: nowMs + Math.min(2_000 * 2 ** Math.min(recovery.attempts, 4), 30_000), + }, + }); + } const failedOrInterrupted = outcome?.status === "failed" || outcome?.status === "interrupted"; - let cursor: string | undefined; - let finalReplySeen = false; - const seen = new Set(); + const expired = !!active?.resultRecovery && nowMs >= active.resultRecovery.deadlineMs; + const pending = !failedOrInterrupted + ? yield* options.client.pending(binding).pipe( + Effect.timeout("3 seconds"), + Effect.catch((cause) => + expired || !active ? Effect.succeed(null) : Effect.fail(cause), + ), + wire, + ) + : null; + if (pending && active) { + for (const interaction of [...pending.questions, ...pending.permissions]) + yield* ask(interaction); + const stillPending = new Set( + [...pending.questions, ...pending.permissions].map((entry) => entry.id), + ); + for (const entry of requests.values()) + if (entry.runtime.status === "pending" && !stillPending.has(entry.native.id)) + yield* resolveRequest(entry); + } + const hasPending = !!pending && pending.questions.length + pending.permissions.length > 0; + if (hasPending && active?.resultRecovery) { + // Human interaction is not a missing-result failure. Persist a fresh + // retrieval window while the customer API confirms it is outstanding. + yield* save({ + ...active, + resultRecovery: { + ...active.resultRecovery, + deadlineMs: Math.max( + active.resultRecovery.deadlineMs, + nowMs + (options.resultRecoveryTimeoutMs ?? 300_000), + ), + }, + }); + } + if (expired) { + if (hasPending) { + yield* status( + "Kilo Cloud has an outstanding interaction. Remote completion does not resolve it.", + ); + } else { + yield* save({ ...active!, resultStatus: "unavailable" }); + yield* finish( + "failed", + "Kilo Cloud completed remotely, but its correlated result could not be retrieved before the recovery deadline. Reopen history to retrieve a late result; no task was resubmitted.", + ).pipe(Effect.uninterruptible); + } + return; + } + // A terminal local run may still receive a late result. Explicit history + // refresh advances the same durable cursor without restarting the run. + let target = active ?? intents.at(-1); + const recovery = + target?.resultRecovery ?? + (target && !active + ? { + deadlineMs: nowMs, + nextAttemptMs: nowMs, + attempts: 0, + cursor: null, + seenCursors: [], + completeReplySeen: false, + incompleteReplySeen: false, + } + : undefined); + let cursor: string | undefined = recovery?.cursor ?? undefined; + let completeReplySeen = recovery?.completeReplySeen ?? false; + let incompleteReplySeen = recovery?.incompleteReplySeen ?? false; + let scanComplete = false; + let resetScan = false; + const seen = new Set(recovery?.seenCursors ?? []); const readHistory = Effect.gen(function* () { - do { + for (let pages = 0; pages < 4; pages++) { const page = yield* wire(options.client.history(binding!, cursor)); - if (page.history === null) break; + if (page.history === null) return; if (page.history.omittedItemCount > 0) - return yield* error( - "Kilo Cloud history is incomplete; remote task state is still unknown.", - ); + return yield* error("Kilo Cloud result history is incomplete."); for (const message of page.history.messages) { yield* project(message, intents); - if ( - message.info.parentID === expectedMessageId && + if (message.info.role !== "assistant" || message.info.parentID !== target?.messageId) + continue; + const complete = message.info.time.completed !== undefined && - message.info.finish && - message.info.finish !== "tool-calls" - ) - finalReplySeen = true; + message.parts.every( + (part) => + part.type !== "tool" || + ["completed", "error"].includes(String(part.state?.status)), + ); + if (complete) completeReplySeen = true; + else incompleteReplySeen = true; } cursor = active && - page.history.messages.some((message) => message.info.id === expectedMessageId) + page.history.messages.some((message) => message.info.id === target?.messageId) ? undefined : (page.history.nextCursor ?? undefined); - if (cursor && seen.has(cursor)) - return yield* error("Kilo Cloud returned a repeated history cursor."); - if (cursor) seen.add(cursor); - } while (cursor); - needsHistoryRestore = false; + if (!cursor) { + scanComplete = true; + return; + } + if (seen.has(cursor) || seen.size >= 100) { + resetScan = true; + return yield* error( + "Kilo Cloud result history exceeded its cursor budget or repeated a cursor.", + ); + } + seen.add(cursor); + } }); if (failedOrInterrupted) { - needsHistoryRestore = true; - // Confirmed termination survives unavailable bootstrap history, but retain any - // output produced while this client was disconnected before closing the turn. yield* readHistory.pipe(Effect.timeout("5 seconds"), Effect.ignore); yield* finish(outcome.status as "failed" | "interrupted").pipe(Effect.uninterruptible); return; } - yield* readHistory; - if (active && active.messageId === expectedMessageId) { - const pending = yield* wire(options.client.pending(binding)); - if (active?.messageId !== expectedMessageId) return; - for (const interaction of [...pending.questions, ...pending.permissions]) - yield* ask(interaction); - const stillPending = new Set( - [...pending.questions, ...pending.permissions].map((request) => request.id), + if (recovery) yield* readHistory.pipe(Effect.timeout("5 seconds"), Effect.ignore); + else yield* readHistory; + const available = + scanComplete && + completeReplySeen && + !incompleteReplySeen && + pending !== null && + !hasPending; + if (target && recovery) { + const updated = { + ...target, + ...(available ? { resultStatus: "available" as const } : {}), + resultRecovery: { + ...recovery, + cursor: scanComplete || resetScan ? null : (cursor ?? null), + seenCursors: scanComplete || resetScan ? [] : [...seen], + completeReplySeen: scanComplete || resetScan ? false : completeReplySeen, + incompleteReplySeen: scanComplete || resetScan ? false : incompleteReplySeen, + }, + }; + // Use the latest revision after reserving this attempt above. + target = active + ? yield* save({ ...updated, revision: active.revision }) + : yield* wire(options.journal.save(updated)); + if (!active) resultStatus = target.resultStatus; + } + if (active?.remoteState === "completed" && available) { + yield* finish("completed").pipe(Effect.uninterruptible); + } else if (active?.state === "awaiting_result") { + yield* status( + hasPending + ? "Kilo Cloud has an outstanding interaction. Remote completion does not resolve it." + : "Kilo Cloud completed remotely. Awaiting its correlated result; Stop cancels result retrieval only.", ); - for (const entry of requests.values()) { - if (entry.runtime.status === "pending" && !stillPending.has(entry.native.id)) { - yield* resolveRequest(entry); - } - } - const result = outcome; - if (active?.messageId !== expectedMessageId) return; - if (result) taskState = result.status; - if ( - result && - result.status !== "queued" && - result.status !== "running" && - (result.status !== "completed" || finalReplySeen) - ) - yield* finish(result.status).pipe(Effect.uninterruptible); } + needsHistoryRestore = active !== undefined && !scanComplete && !!cursor; }); const lifecycle = Effect.gen(function* () { if (!thread || !binding) return; @@ -736,6 +868,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { sessionId: binding.cloudAgentSessionId, worktreeId: binding.worktreeId, task: taskState, + ...(resultStatus ? { result: resultStatus } : {}), sandbox: sandbox?.status ?? ("unknown" as const), billing: billing?.phase ?? ("unknown" as const), billingAttribution: billing?.attribution ?? null, @@ -820,7 +953,9 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { .pipe( Effect.catch(() => status( - "Cloud connection unavailable. Task and billing status are unknown; no prompt was resubmitted.", + taskState === "completed" + ? "Cloud result retrieval is unavailable. Remote execution completed; sandbox and billing are separate. No prompt was resubmitted." + : "Cloud connection unavailable. Task and billing status are unknown; no prompt was resubmitted.", ), ), ); @@ -909,10 +1044,20 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ) return yield* error("Cloud journal belongs to another account or repository."); binding = last?.binding ?? undefined; - taskState = last?.state === "active" ? "unknown" : (last?.state ?? "not_started"); + taskState = + last?.remoteState ?? + (last?.state === "awaiting_result" + ? "completed" + : last?.state === "active" + ? "unknown" + : (last?.state ?? "not_started")); + resultStatus = last?.resultStatus; monitorSandbox = !!last?.binding; active = - last && (last.state === "active" || last.state === "admission_unknown") + last && + (last.state === "active" || + last.state === "admission_unknown" || + last.state === "awaiting_result") ? last : undefined; if (binding) thread = { ...thread, nativeThreadRef: nativeRef(binding.kiloSessionId) }; @@ -973,9 +1118,19 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { saved.state === "failed" || saved.state === "interrupted" ) { + active = undefined; needsHistoryRestore = true; yield* reconcile().pipe(Effect.timeout("10 seconds"), Effect.ignore); - if (active) yield* finish(saved.state); + active = + (yield* wire(options.journal.readThread(thread!.id))).find( + (entry) => entry.operationKey === saved.operationKey, + ) ?? saved; + yield* finish( + saved.state, + saved.resultStatus === "unavailable" + ? "Kilo Cloud completed remotely, but its result was unavailable before the recovery deadline." + : undefined, + ); } yield* watch; return; @@ -1080,17 +1235,20 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { }; if (!(yield* wire(options.journal.reserve(intent)))) return yield* error("A previous cloud admission still needs reconciliation."); + resultStatus = undefined; active = intent; taskState = "admission_unknown"; monitorSandbox = true; thread = providerThread; if (thread.nativeMetadata?.cloudExecution) { + const cloudExecution = { ...thread.nativeMetadata.cloudExecution }; + delete cloudExecution.result; thread = { ...thread, nativeMetadata: { ...thread.nativeMetadata, cloudExecution: { - ...thread.nativeMetadata.cloudExecution, + ...cloudExecution, task: taskState, sandbox: "unknown", billing: "unknown", @@ -1149,6 +1307,14 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { Effect.gen(function* () { yield* owned(request.providerThread); if (!active || active.providerTurn.id !== request.providerTurnId) return false; + if (active.state === "awaiting_result") { + yield* save({ ...active, resultStatus: "cancelled" }); + yield* finish("interrupted"); + yield* status( + "Result retrieval cancelled locally. Kilo already reported completion; sandbox and billing were not stopped.", + ); + return false; + } if (!binding) return yield* error( "Cloud admission has no confirmed session ID. Remote Stop is unavailable; task and billing status remain unknown.", @@ -1200,10 +1366,13 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { Effect.timeout("10 seconds"), Effect.mapError(() => error( - "Cloud history is temporarily unavailable. Remote execution may still be active.", + taskState === "completed" + ? "Cloud history is unavailable. Remote execution completed; sandbox and billing are separate." + : "Cloud history is temporarily unavailable. Remote execution may still be active.", ), ), ); + yield* lifecycle; const intents = yield* wire(options.journal.readThread(thread!.id)); return { providerThread: thread!, diff --git a/apps/server/src/provider/Drivers/KiloDriver.ts b/apps/server/src/provider/Drivers/KiloDriver.ts index 397a4e0bef9d..8a97508fcbf0 100644 --- a/apps/server/src/provider/Drivers/KiloDriver.ts +++ b/apps/server/src/provider/Drivers/KiloDriver.ts @@ -20,6 +20,8 @@ import type { ProviderDriver } from "../ProviderDriver.ts"; import { buildServerProvider } from "../providerSnapshot.ts"; import { makeManualOnlyProviderMaintenanceCapabilities } from "../providerMaintenance.ts"; +const isRuntimeError = Schema.is(KiloRuntime.KiloRuntimeError); + const decode = Schema.decodeSync(KiloSettings); const kind = KiloAdapter.KILO_PROVIDER; const systemKeys = new Set([ @@ -162,7 +164,8 @@ export const KiloDriver: ProviderDriver = { version: null, status: "warning", auth: { status: "unknown", profileId: input.config.accountId }, - message: KiloRuntime.localExecutionBlocked.message, + message: + "Checking Kilo. Native configuration and MCP servers are trusted; tool approvals are not a sandbox.", }, }), ); @@ -239,7 +242,7 @@ export const KiloDriver: ProviderDriver = { ...(models.length ? { message: - "Kilo is ready. Model availability and authentication have not been verified by running a prompt. Subagents require Full access mode; Kilo-configured approvals still apply.", + "Kilo is ready. Native configuration and MCP servers are trusted; tool approvals are not a sandbox. Model authentication has not been prompt-tested. Subagents require Full access.", } : { message: `No connected models in the Kilo account profile ${profileDirectory}. Configure this profile with Kilo or set explicit provider environment variables.`, @@ -249,12 +252,14 @@ export const KiloDriver: ProviderDriver = { ); }).pipe(Effect.scoped); latest = yield* check.pipe( - Effect.catch(() => + Effect.catch((cause) => Effect.succeed({ ...latest, status: "error" as const, installed: false, - message: KiloRuntime.localExecutionBlocked.message, + message: isRuntimeError(cause) + ? cause.message + : "Could not refresh Kilo. Check the binary, account profile and native configuration.", }), ), ); diff --git a/apps/server/src/provider/kilo/KiloCloudJournal.ts b/apps/server/src/provider/kilo/KiloCloudJournal.ts index 24ca12c08afa..1d1635ea5072 100644 --- a/apps/server/src/provider/kilo/KiloCloudJournal.ts +++ b/apps/server/src/provider/kilo/KiloCloudJournal.ts @@ -29,7 +29,31 @@ export const CloudIntent = Schema.Struct({ kiloSessionId: Schema.NonEmptyString, }), ), - state: Schema.Literals(["admission_unknown", "active", "completed", "failed", "interrupted"]), + state: Schema.Literals([ + "admission_unknown", + "active", + "awaiting_result", + "completed", + "failed", + "interrupted", + ]), + remoteState: Schema.optional( + Schema.Literals(["queued", "running", "completed", "failed", "interrupted"]), + ), + resultStatus: Schema.optional( + Schema.Literals(["awaiting_result", "available", "unavailable", "cancelled"]), + ), + resultRecovery: Schema.optional( + Schema.Struct({ + deadlineMs: Schema.Number, + nextAttemptMs: Schema.Number, + attempts: Schema.Number, + cursor: Schema.NullOr(Schema.String), + seenCursors: Schema.Array(Schema.String), + completeReplySeen: Schema.optional(Schema.Boolean), + incompleteReplySeen: Schema.optional(Schema.Boolean), + }), + ), interruptRequested: Schema.Boolean, answeredRequestIds: Schema.Array(Schema.String), providerThread: OrchestrationV2ProviderThread, @@ -71,7 +95,7 @@ export const make = Effect.fn("KiloCloudJournal.make")(function* (directory: str yield* sql`PRAGMA synchronous = FULL`; yield* sql`CREATE TABLE IF NOT EXISTS intents (operation_key TEXT PRIMARY KEY, thread_id TEXT NOT NULL, state TEXT NOT NULL, body TEXT NOT NULL)`; yield* sql`CREATE INDEX IF NOT EXISTS cloud_intents_thread ON intents(thread_id)`; - yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS one_active_cloud_intent ON intents(thread_id) WHERE state IN ('active', 'admission_unknown')`; + yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS one_active_cloud_intent_v2 ON intents(thread_id) WHERE state IN ('active', 'admission_unknown', 'awaiting_result')`; }).pipe(Effect.mapError((cause) => new CloudJournalError({ operation: "write", cause }))); const read = Effect.gen(function* () { const rows = yield* sql<{ body: string }>`SELECT body FROM intents ORDER BY rowid`; diff --git a/apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs b/apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs new file mode 100644 index 000000000000..f24ba880dfed --- /dev/null +++ b/apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs @@ -0,0 +1,43 @@ +import * as Effect from "effect/Effect"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import * as Runtime from "./KiloRuntime.ts"; + +// Report the owned PID immediately for emergency cleanup; readiness is separate. +await Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + let pid; + const runtime = yield* Runtime.make({ + instanceId: "crash-fixture", + binaryPath: process.argv[2], + profileDirectory: process.argv[3], + environment: { + PATH: process.env.PATH, + HOME: process.argv[3], + KILO_DISABLE_MODELS_FETCH: "1", + KILO_DISABLE_DEFAULT_PLUGINS: "1", + KILO_DISABLE_EXTERNAL_SKILLS: "1", + }, + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, { + ...spawner, + spawn: (...args) => + spawner.spawn(...args).pipe( + Effect.tap((child) => + Effect.sync(() => { + pid = Number(child.pid); + process.send({ type: "spawned", pid }); + }), + ), + ), + }), + ); + const connection = yield* runtime.open(process.argv[3]); + const session = yield* connection.client.create([]); + process.send({ type: "ready", pid, session }); + yield* Effect.never; + }).pipe(Effect.provide(NodeServices.layer)), + ), +); diff --git a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts index 1142b2693e87..ea9c93997290 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts @@ -1,3 +1,8 @@ +// @effect-diagnostics nodeBuiltinImport:off - real owner crash fixture. +import * as NodeChildProcess from "node:child_process"; +import * as NodeEvents from "node:events"; +import * as NodeURL from "node:url"; +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { ProviderInstanceId, ProviderSessionId, ThreadId } from "@t3tools/contracts"; @@ -16,6 +21,15 @@ import * as ServerConfig from "../../config.ts"; import * as IdAllocator from "../../orchestration-v2/IdAllocator.ts"; const binary = process.env.KILO_BIN; +const platform = HostProcessPlatform.defaultValue(); +const decodeGroup = Schema.decodeUnknownEffect( + Schema.fromJsonString(Schema.Struct({ pgid: Schema.Number })), +); +const decodeOwner = Schema.decodeUnknownEffect( + Schema.fromJsonString( + Schema.Struct({ owner: Schema.Struct({ pid: Schema.Number }), pgid: Schema.Number }), + ), +); const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const environment = { PATH: process.env.PATH, @@ -29,10 +43,9 @@ const environment = { KILO_DISABLE_PROJECT_CONFIG: "1", }; -// Historical native conformance coverage. Re-enable only after an audited MCP runtime fix. -describe.skip("KiloRuntime native lifecycle", () => { +describe.skipIf(!binary)("KiloRuntime native lifecycle", () => { it.live( - "does not execute repository or external plugins before session permissions", + "loads explicitly trusted repository and external plugins before tool approvals", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -44,17 +57,18 @@ describe.skip("KiloRuntime native lifecycle", () => { const marker = path.join(root, "repository-plugin-ran"); const explicitMarker = path.join(root, "explicit-plugin-ran"); const body = (target: string) => - `import { writeFileSync } from "node:fs";\nwriteFileSync(${encodeJson(target)}, "executed");\nexport const fixture = async () => ({});\n`; + `import { writeFileSync } from "node:fs";\nimport { spawn } from "node:child_process";\nconst child = spawn(${encodeJson(process.execPath)}, ["-e", "setInterval(()=>{},1000)"], {stdio:"ignore"});\nwriteFileSync(${encodeJson(target)}, String(child.pid));\nexport const fixture = async () => ({});\n`; yield* fs.writeFileString(path.join(pluginDir, "unsafe.ts"), body(marker)); const explicitPlugin = path.join(root, "explicit.ts"); yield* fs.writeFileString(explicitPlugin, body(explicitMarker)); - // A profile override must not reopen the approval bypass. + // Native configuration is explicitly trusted, independently of tool approvals. const runtime = yield* KiloRuntime.make({ instanceId: "plugins", binaryPath: binary!, profileDirectory: path.join(root, "profile"), environment: { ...environment, + HOME: root, KILO_DISABLE_PROJECT_CONFIG: "0", KILO_PURE: "0", KILO_CONFIG_CONTENT: encodeJson({ plugin: [explicitPlugin] }), @@ -66,8 +80,30 @@ describe.skip("KiloRuntime native lifecycle", () => { { permission: "*", pattern: "*", action: "ask" }, ]); assert.equal((yield* connection.client.read(ref)).id, ref.sessionId); - assert.isFalse(yield* fs.exists(marker)); - assert.isFalse(yield* fs.exists(explicitMarker)); + assert.isTrue(yield* fs.exists(marker)); + assert.isTrue(yield* fs.exists(explicitMarker)); + if (platform === "linux") { + const ledgerDir = path.join(root, "profile", "t3-processes", "opencode-servers"); + const entry = (yield* fs.readDirectory(ledgerDir))[0]!; + const recorded = yield* decodeGroup( + yield* fs.readFileString(path.join(ledgerDir, entry)), + ); + const descendants = [ + Number(yield* fs.readFileString(marker)), + Number(yield* fs.readFileString(explicitMarker)), + ]; + const running = (pid: number) => + fs.readFileString(`/proc/${pid}/stat`).pipe( + Effect.map((stat) => !stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")), + Effect.orElseSucceed(() => false), + ); + for (const pid of descendants) assert.isTrue(yield* running(pid)); + // Kill only the recorded owned leader while its session is idle. Exit + // observation must clean up descendants without an active-turn error. + process.kill(recorded.pgid, "SIGKILL"); + yield* connection.exitCode; + for (const pid of descendants) assert.isFalse(yield* running(pid)); + } }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), { timeout: 30000 }, ); @@ -88,7 +124,7 @@ describe.skip("KiloRuntime native lifecycle", () => { instanceId: "account-test", binaryPath: binary!, profileDirectory: root, - environment, + environment: { ...environment, HOME: root }, }); const connection = yield* runtime.open(root); const native = yield* connection.client.create([]); @@ -97,7 +133,7 @@ describe.skip("KiloRuntime native lifecycle", () => { displayName: undefined, enabled: false, config: { ...KiloDriver.defaultConfig(), binaryPath: binary!, profileDirectory: root }, - environment: Object.entries(environment).flatMap(([name, value]) => + environment: Object.entries({ ...environment, HOME: root }).flatMap(([name, value]) => value === undefined ? [] : [{ name, value, sensitive: false }], ), }); @@ -183,13 +219,13 @@ describe.skip("KiloRuntime native lifecycle", () => { instanceId: "personal", binaryPath: binary!, profileDirectory: path.join(root, "personal"), - environment, + environment: { ...environment, HOME: root }, }).pipe(Effect.provideService(Scope.Scope, accountScope)); const work = yield* KiloRuntime.make({ instanceId: "work", binaryPath: binary!, profileDirectory: path.join(root, "work-account"), - environment, + environment: { ...environment, HOME: root }, }); const sessionScope = yield* Scope.fork(yield* Effect.scope); const first = yield* runtime @@ -215,6 +251,97 @@ describe.skip("KiloRuntime native lifecycle", () => { { timeout: 30000 }, ); + it.live.skipIf(platform !== "linux")( + "reaps a real Kilo process after its T3 owner is killed and resumes its session", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const profile = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-crash-" }); + let group: number | undefined; + const owner = yield* Effect.acquireRelease( + Effect.sync(() => + NodeChildProcess.spawn( + process.execPath, + [ + NodeURL.fileURLToPath(new URL("./KiloRuntime.crash.fixture.mjs", import.meta.url)), + binary!, + profile, + ], + { stdio: ["ignore", "ignore", "ignore", "ipc"] }, + ), + ), + (child) => + Effect.sync(() => { + child.kill("SIGKILL"); + if (group !== undefined) { + try { + process.kill(-group, "SIGKILL"); + } catch { + /* already stopped */ + } + } + }), + ); + const message = yield* Effect.promise( + () => + new Promise<{ + pid: number; + session: { instanceId: string; sessionId: string; directory: string }; + }>((resolve, reject) => { + owner.on("message", (value) => { + const message = value as { + type: string; + pid: number; + session: { instanceId: string; sessionId: string; directory: string }; + }; + group = message.pid; + if (message.type === "ready") resolve(message); + }); + owner.once("exit", () => + reject(new Error("Kilo crash fixture exited before readiness")), + ); + owner.once("error", reject); + }), + ); + const entries = yield* fs.readDirectory( + path.join(profile, "t3-processes", "opencode-servers"), + ); + assert.equal(entries.length, 1); + const recorded = yield* decodeOwner( + yield* fs.readFileString( + path.join(profile, "t3-processes", "opencode-servers", entries[0]!), + ), + ); + assert.equal(recorded.owner.pid, owner.pid); + assert.equal(recorded.pgid, group); + const exited = NodeEvents.EventEmitter.once(owner, "exit"); + owner.kill("SIGKILL"); + yield* Effect.promise(() => exited); + const running = (pid: number) => + fs.readFileString(`/proc/${pid}/stat`).pipe( + Effect.map((stat) => !stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")), + Effect.orElseSucceed(() => false), + ); + assert.isTrue(yield* running(message.pid)); + const restarted = yield* KiloRuntime.make({ + instanceId: "crash-fixture", + binaryPath: binary!, + profileDirectory: profile, + environment: { ...environment, HOME: profile }, + }); + assert.isFalse(yield* running(message.pid)); + assert.deepEqual( + yield* fs.readDirectory(path.join(profile, "t3-processes", "opencode-servers")), + [], + ); + const fresh = yield* restarted.open(profile); + assert.equal((yield* fresh.client.read(message.session)).id, message.session.sessionId); + assert.isTrue(yield* fresh.isRunning); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + { timeout: 30000 }, + ); + it.live( "cleans failed startup and can open a fresh process afterward", () => @@ -226,7 +353,7 @@ describe.skip("KiloRuntime native lifecycle", () => { instanceId: "broken", binaryPath: path.join(root, "missing"), profileDirectory: root, - environment, + environment: { ...environment, HOME: root }, }); const failure = yield* bad.open(root).pipe(Effect.flip); assert.equal(failure.operation, "spawn"); @@ -240,16 +367,17 @@ describe.skip("KiloRuntime native lifecycle", () => { instanceId: "early", binaryPath: earlyExit, profileDirectory: root, - environment, + environment: { ...environment, HOME: root }, }); const earlyFailure = yield* exiting.open(root).pipe(Effect.flip); assert.equal(earlyFailure.operation, "startup"); assert.notInclude(earlyFailure.message, "do-not-leak"); + assert.include(earlyFailure.message, "code 7"); const good = yield* KiloRuntime.make({ instanceId: "working", binaryPath: binary!, profileDirectory: root, - environment, + environment: { ...environment, HOME: root }, }); const connection = yield* good.open(root); assert.isTrue(yield* connection.isRunning); diff --git a/apps/server/src/provider/kilo/KiloRuntime.safety.test.ts b/apps/server/src/provider/kilo/KiloRuntime.safety.test.ts deleted file mode 100644 index aabdc4dc1038..000000000000 --- a/apps/server/src/provider/kilo/KiloRuntime.safety.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; -import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Schema from "effect/Schema"; -import * as Path from "effect/Path"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import * as KiloRuntime from "./KiloRuntime.ts"; - -const encode = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); - -it.live("rejects every local open before executing the binary or changing config", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-safety-" }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - let spawned = 0; - const observedSpawner = { - ...spawner, - spawn: (...args: Parameters) => { - spawned++; - return spawner.spawn(...args); - }, - }; - const profile = path.join(root, "profile"); - const fixtures = [ - ".kilo/mcp.json", - ".kilocode/mcp.json", - "profile/config/kilo/kilo.json", - ".kilo/kilo.json", - ]; - const sources = new Map(); - for (const [index, file] of fixtures.entries()) { - const target = path.join(root, file); - const command = [ - "node", - "-e", - `require('node:fs').writeFileSync(${encode(path.join(root, "marker"))}, 'executed')`, - ]; - const contents = encode( - file.endsWith("mcp.json") - ? { - mcpServers: { - [`new-server-${index}`]: { command: command[0], args: command.slice(1) }, - }, - } - : { - mcp: { [`new-server-${index}`]: { type: "local", command } }, - }, - ); - yield* fs.makeDirectory(path.dirname(target), { recursive: true }); - yield* fs.writeFileString(target, contents); - sources.set(target, contents); - } - const runtime = yield* KiloRuntime.make({ - instanceId: "safety-a", - binaryPath: process.env.KILO_BIN ?? "kilo", - profileDirectory: profile, - environment: { - PATH: process.env.PATH, - KILO_PURE: "0", - KILO_DISABLE_PROJECT_CONFIG: "0", - KILO_PLATFORM: "vscode", - KILOCODE_FEATURE: "daemon", - }, - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, observedSpawner)); - for (const directory of [root, path.join(root, "nested"), root]) { - const failure = yield* runtime.open(directory).pipe(Effect.flip); - assert.equal(failure.operation, "runtime-safety"); - } - // A later new server name and account change cannot open a second entry path. - yield* fs.writeFileString( - path.join(root, ".kilocode/mcp.json"), - '{"mcpServers":{"later-server":{"url":"http://127.0.0.1:9"}}}', - ); - sources.delete(path.join(root, ".kilocode/mcp.json")); - const failure = yield* runtime.open(root).pipe(Effect.flip); - assert.equal(failure.operation, "runtime-safety"); - assert.equal(spawned, 0); - assert.isFalse(yield* fs.exists(path.join(root, "marker"))); - for (const [target, contents] of sources) - assert.equal(yield* fs.readFileString(target), contents); - // Credential selection still works, without starting a native process. - const authDir = path.join(profile, "data/kilo"); - yield* fs.makeDirectory(authDir, { recursive: true }); - yield* fs.writeFileString( - path.join(authDir, "auth.json"), - '{"kilo":{"type":"api","key":"synthetic-a"}}', - ); - const first = yield* KiloRuntime.readAuth(profile, {}); - yield* fs.writeFileString( - path.join(authDir, "auth.json"), - '{"kilo":{"type":"api","key":"synthetic-b"}}', - ); - assert.notEqual(first, yield* KiloRuntime.readAuth(profile, {})); - assert.equal(yield* KiloRuntime.readAuth(profile, { KILO_AUTH_CONTENT: first }), first); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), -); diff --git a/apps/server/src/provider/kilo/KiloRuntime.ts b/apps/server/src/provider/kilo/KiloRuntime.ts index 2b1c391059a0..9a5429023824 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.ts @@ -15,6 +15,7 @@ import { resolveSpawnCommand } from "@t3tools/shared/shell"; import { signalProcessGroup } from "../../process/processGroup.ts"; import * as KiloSessionClient from "./KiloSessionClient.ts"; +import * as ServerLedger from "../OpenCodeServerLedger.ts"; export class KiloRuntimeError extends Schema.TaggedError()("KiloRuntimeError", { operation: Schema.String, @@ -43,6 +44,8 @@ export class KiloRuntime extends Context.Service< } >()("t3/provider/kilo/KiloRuntime") {} +const isRuntimeError = Schema.is(KiloRuntimeError); + const authSchema = Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)); const decodeAuth = Schema.decodeUnknownEffect(authSchema); const encodeAuth = Schema.encodeEffect(authSchema); @@ -81,17 +84,6 @@ export const readAuth = Effect.fn("KiloRuntime.readAuth")(function* ( ); }); -// No released runtime has a verified pre-connect MCP policy boundary. This gate -// deliberately has no environment/config override. Restore execution only with -// an audited dependency and real-process startup/reload/reconnect tests. -export const localExecutionBlocked = new KiloRuntimeError({ - operation: "runtime-safety", - detail: - "Local Kilo execution is disabled: CLI 7.8.3 can start MCP commands and connections before approval. A verified runtime fix is required.", -}); -const requireSafeRuntime: Effect.Effect = - Effect.fail(localExecutionBlocked); - /** Every open owns a process. Registry replacement closes the old account's process scopes. */ export const make = Effect.fn("KiloRuntime.make")(function* (input: { readonly instanceId: string; @@ -110,6 +102,8 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { const fail = (operation: string, detail: string) => (cause: unknown) => new KiloRuntimeError({ operation, detail, cause }); const profile = path.resolve(input.profileDirectory); + const ledger = yield* ServerLedger.make({ stateDir: path.join(profile, "t3-processes") }); + yield* ledger.reapOrphans; const authContent = input.authContent ?? (yield* readAuth(profile, input.environment)); const environment: NodeJS.ProcessEnv = { ...input.environment, @@ -118,10 +112,8 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { XDG_CACHE_HOME: path.join(profile, "cache"), XDG_STATE_HOME: path.join(profile, "state"), KILO_DISABLE_AUTOUPDATE: "1", - // Repository config and external plugins execute before session permissions. - // Keep these forced after instance overrides, including in Full access. - KILO_DISABLE_PROJECT_CONFIG: "1", - KILO_PURE: "1", + // Native configuration is trusted, as with OpenCode. Tool approvals do not + // sandbox plugins or MCP initialization, including legacy configuration. // Background children outlive root turns and need a separate T3 continuation contract. KILO_EXPERIMENTAL_BACKGROUND_SUBAGENTS: "false", KILO_SERVER_USERNAME: "kilo", @@ -157,7 +149,6 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { // Readiness output can contain project/plugin diagnostics; never include it in client errors. return KiloRuntime.of({ open: Effect.fn("KiloRuntime.open")(function* (directory) { - yield* requireSafeRuntime; if (closed) return yield* new KiloRuntimeError({ operation: "open", @@ -183,6 +174,8 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { ["serve", "--hostname=127.0.0.1", "--port=0"], { env: environment, extendEnv: false }, ); + // Forget only after the owned group is stopped, including failed readiness. + const ledgerScope = yield* Scope.fork(scope); const child = yield* spawner .spawn( ChildProcess.make(command.command, command.args, { @@ -224,6 +217,12 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { ), ); yield* Effect.addFinalizer(() => cleanup); + const forget = yield* ledger.track({ + pid: Number(child.pid), + port: 0, + args: ["serve", "--hostname=127.0.0.1", "--port=0"], + }); + yield* Scope.addFinalizer(ledgerScope, forget); const guard = checkAuth.pipe(Effect.onError(() => cleanup)); // Observe idle or in-flight account replacement as well as request boundaries. // Never read credential files once per SSE event. @@ -247,6 +246,9 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { const exitCode = child.exitCode.pipe( Effect.map(Number), Effect.orElseSucceed(() => -1), + // Native exit can leave configured MCP/plugin children in the group, + // including when no T3 turn is active. The cached cleanup owns that group. + Effect.tap(() => cleanup), ); yield* exitCode.pipe( Effect.flatMap((code) => @@ -262,8 +264,13 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { ); const url = yield* Deferred.await(ready).pipe( Effect.timeout("30 seconds"), - Effect.mapError( - fail("startup", "Kilo did not become ready. Check its installation and configuration."), + Effect.mapError((cause) => + isRuntimeError(cause) + ? cause + : fail( + "startup", + "Kilo did not become ready. Check its installation and configuration.", + )(cause), ), ); const client = yield* KiloSessionClient.make({ diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 017fe116852f..727b7feb1944 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -10723,8 +10723,9 @@ export default function ChatView(props: ChatViewProps) { {cloudExecution ? (

Last observation: {cloudExecution.observedAt ?? "unavailable"}. Task:{" "} - {cloudExecution.task}. Sandbox: {cloudExecution.sandbox}. Compute:{" "} - {cloudExecution.billing} + {cloudExecution.task}.{" "} + {cloudExecution.result ? `Result: ${cloudExecution.result}. ` : ""}Sandbox:{" "} + {cloudExecution.sandbox}. Compute: {cloudExecution.billing} {cloudExecution.billingAttribution === "payer_shared" ? " (shared account)" : ""} diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md index f09edf4740ef..834dd6e65dfb 100644 --- a/docs/user/providers-kilo.md +++ b/docs/user/providers-kilo.md @@ -1,66 +1,89 @@ # Kilo -Local Kilo execution is disabled in this preview. T3 refuses every local process -start, including provider checks, new prompts, text generation and restored -sessions. CLI 7.8.3 can start MCP commands and connections before approval despite -pure mode, disabled project configuration and deny-all session permissions. Both -legacy project directories and global/profile sources are affected. The Kilo tool -sandbox does not cover MCP startup or reconnect. - -No released runtime has a verified fix. Restoring local support requires an -explicitly supported dependency with a process-level MCP policy boundary, tested -across startup, prompts, configuration changes, reconnect and resume. A profile -scan or disabling known server names is insufficient. T3 does not alter your -configuration or patch the installed CLI. There is no unsafe-execution override. -Existing local history and account configuration are retained. The earlier local -prompt, approval, fork, rewind, subagent and text-generation tests are historical -conformance evidence, not currently available functionality. Local Kilo and cloud -parallel execution is consequently blocked. Restricted-mode subagents remain -disallowed even if local execution is restored. - -| Capability | Local Kilo | Kilo Cloud | -| ------------------------------------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------------------ | -| New prompts and follow-up | Blocked before process start | Full access only | -| Concurrent isolated threads | Blocked | Separate remote worktrees and task identities | -| Local/cloud parallel tasks | Blocked | Requires a safe local runtime | -| History and recovery | Stored history retained; native resume blocked | Durable task identity, paginated admission recovery; no blind resubmit | -| Stop | No local process starts | Requests inference interruption; sleep and compute observed separately | -| Approvals and questions | Blocked | Handles interactions emitted by the remote runtime; cannot enforce restricted policy | -| Rewind, fork, local files, checkpoints, text generation | Blocked | Not supported | -| Subagents | Blocked | Remote Full access may execute them; child history not integrated | -| Web, desktop and mobile | Shows execution-blocked status | Selection, account/model settings and task status; local workspace controls disabled | - -For remote execution, add a separate **Kilo Cloud** instance in Settings > Providers. -Select a profile signed in through the official Kilo login, a GitHub repository that -account can access, its branch, and a model. Enable paid cloud execution only when -you want prompts and that repository sent to Kilo. T3 never uploads your local -checkout or uncommitted changes. Each cloud thread has its own remote worktree. Local Kilo concurrency awaits a -safe native runtime. - -Cloud execution currently requires **Full access**. The deployed cloud runtime -does not apply custom agent permissions, so T3 refuses restricted and Plan modes -before submitting a paid task. Shell, edits and subagents cannot be restricted in -cloud Full access. Cloud subagent history is not integrated. Local Kilo is also unavailable while the runtime safety gate is in place. Inherited Kilo profiles with setup, +Add **Kilo** in Settings > Providers, select the Kilo CLI 7.8.3 binary and an +account profile, then refresh the provider. Profiles isolate credentials and native +session storage. Sign in with the official Kilo CLI separately; T3 never signs in +automatically. Changing credentials retires the old runtime and requires a new +thread. Saved history remains available. + +## Local configuration and approvals + +Like T3's OpenCode provider, Kilo trusts native runtime configuration, plugins and +MCP servers. Only open repositories and profiles whose configuration you trust. +Configured MCP processes or connections can start before a model tool call or +approval. Supervised and Plan modes govern supported tool calls; they are not an +OS sandbox and do not isolate native configuration or MCP initialization. + +Kilo 7.8.3 loads legacy `.kilo/mcp.json` and `.kilocode/mcp.json` even when +`KILO_DISABLE_PROJECT_CONFIG` is enabled. `KILO_PURE` suppresses external plugins, +not all MCP loading. T3 does not force either flag as a security boundary, rewrite +configuration, or patch the installed runtime. Explicit native settings remain +trusted. Background subagents stay disabled. Foreground child agents require Full +access because Kilo does not inherit parent `ask` rules reliably. + +T3 stops owned processes on normal shutdown. On Linux and macOS, it records their +process identity and reaps processes from a dead T3 owner when the same profile +is reopened. Cleanup checks the recorded PID, start time, command and owner; +it never kills by executable name. Crash recovery on macOS and native Windows +process cleanup have not been verified in this environment. + +| Capability | Local Kilo | Kilo Cloud | +| ------------------------------------------ | --------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| Prompts and follow-up | Native streaming, tools and reasoning | Full access; history updates, no token-streaming claim | +| Concurrent sessions | Separate processes and account profiles | Separate task identities and remote worktrees; runs alongside local sessions | +| History and recovery | Native history and resume | Durable admission and result recovery; no blind paid resubmission | +| Stop | Native abort and owned-process cleanup | Inference interrupt while running; local retrieval cancellation after remote completion | +| Approvals and questions | Native supported tool approvals and questions | Handles emitted interactions; cannot enforce restricted policy | +| Rewind, fork, checkpoints, text generation | Integrated with native sessions and T3 checkpoints | Not supported | +| Subagents | Foreground Full access only; restricted modes denied | Remote Full access may run them; child history not integrated | +| Clients | Web/desktop/mobile selection, models, status and controls | Account/repository/model settings and task status; local workspace actions disabled | + +## Cloud execution and costs + +Add a separate **Kilo Cloud** instance in Settings > Providers. Select a profile +signed in through the official Kilo login, an accessible GitHub repository, its +branch and a model. Personal accounts are supported. Enabling paid cloud execution +allows prompts and that remote repository to be sent to Kilo. T3 never uploads +local checkout files or uncommitted changes. Each cloud thread has a remote worktree. + +Cloud requires **Full access**. The deployed runtime does not apply custom agent +permissions, so T3 rejects restricted and Plan modes before paid admission. Remote +shell commands, edits and subagents can run. Automatic commits are disabled, but +this is not a read-only execution policy. Profiles with inherited setup commands, MCP, skills, agents or environment variables are rejected before a new cloud task. +The local trust choice does not relax this cloud restriction. + +A cloud thread cannot use local attachments, terminals, Git actions, checkpoints, +rewind, forks or background text generation. Switching accounts does not transfer +existing tasks or stop them. Reconnecting uses the saved task identity. Uncertain +admission is reconciled through paginated customer APIs, never automatically resent. + +Cloud tasks spend Kilo credit for inference and sandbox use. Inference interruption, +a closed stream, remote completion and sandbox sleep are separate events. Task, +result, sandbox and compute status are shown separately. Compute estimates can cover +a shared account sandbox and are not a per-task invoice. Unknown or settling status +does not mean billing has stopped. T3 does not top up credit or force sandbox sleep. + +## Results after remote completion + +The customer `workspace_` API reports execution status separately from history. +T3 marks a completed task `awaiting_result` until it retrieves output correlated to +the original message, account, worktree and native session. A completed, textless +assistant or terminal tool-only outcome is valid; unrelated replies, unfinished +tools and outstanding interactions cannot finish the local turn. + +Retrieval reads at most four pages per attempt, with a 100-cursor cycle limit, +backoff up to 30 seconds and a five-minute recovery window. Progress, next attempt +and deadline survive restart. A confirmed outstanding interaction gives the user +time to respond and renews that window. Missing output after the window produces a +specific local result-retrieval failure while preserving remote `completed`. +Reopening history can retrieve a late result without restarting the failed turn, +duplicating its messages or submitting a new paid task. + +Stop while `awaiting_result` cancels local result retrieval. It does not send a +remote interrupt or claim the sandbox is sleeping. Stop during running inference +requests remote interruption and waits for confirmation; billing remains separate. -Cloud prompts, native history and follow-up messages use the same remote session. -T3 reconnects by its saved task identity and does not automatically resend an -uncertain start. A cloud thread cannot use local attachments, terminals, file -checkpoints, rewind, forks or background text generation. Switching accounts does -not transfer existing tasks or stop them. - -Cloud tasks spend Kilo credit for inference and sandbox use. Automatic commits are -disabled, but an agent in Full access can still modify the remote checkout. Stop -requests inference interruption; closing a stream, task completion and sandbox -sleep are separate events. The thread shows task, sandbox and compute status -separately. Compute estimates can cover a shared account sandbox and are not a -per-task invoice. Unknown or settling status does not mean billing has stopped. -T3 does not top up credit or force a sandbox to sleep. - -Admission recovery retains scan progress and follows customer API cursors. An -unresolved start remains uncertain and is not automatically submitted again. -If the remote API reports completion without the corresponding final reply, -T3 keeps the turn unresolved instead of inventing a successful result. This -recovery case still needs a verified terminal contract; inspect the task in Kilo. -A failed or interrupted task can finish even if its history is incomplete. -Reopening the thread can retry history retrieval. +Local/cloud concurrency and recovery are covered by actual local CLI sessions and +loopback customer-contract tests. These do not replace live verification of every +deployed cloud behavior or native platform testing. diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 3f385c7ce4c0..99eda96cc672 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -798,6 +798,9 @@ export type OrchestrationV2PendingBackgroundTask = typeof OrchestrationV2Pending export const OrchestrationV2ProviderThreadNativeMetadata = Schema.Struct({ cloudExecution: Schema.optional( Schema.Struct({ + result: Schema.optional( + Schema.Literals(["awaiting_result", "available", "unavailable", "cancelled"]), + ), repository: Schema.String, branch: Schema.String, sessionId: Schema.NullOr(Schema.String), diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 011cf2e2f998..7dbde0321ede 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -977,7 +977,8 @@ export const KiloSettings = makeProviderSettingsSchema( binaryPath: makeBinaryPathSetting("kilo").pipe( Schema.annotateKey({ title: "Binary path", - description: "Kilo CLI 7.8.3 executable.", + description: + "Kilo CLI 7.8.3 executable. Native configuration, plugins and MCP servers are trusted. Tool approvals are not an OS sandbox.", providerSettingsForm: { placeholder: "kilo", clearWhenEmpty: "omit" }, }), ), From 53e8d891e8091dddafdbc664cb5c1a7f6e2e2bd2 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sat, 3 Oct 2026 20:02:46 +0000 Subject: [PATCH 16/44] fix(kilo): identify result failures and keep fixtures offline --- .../src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts | 4 +++- .../src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts | 5 +++-- apps/server/src/provider/kilo/KiloRuntime.live.test.ts | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index 6557e49e80bf..0d9d98c4cc8c 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -925,7 +925,9 @@ it.live( assert.isTrue( failure?.type === "turn.terminal" && failure.status === "failed" && - failure.failure?.message.includes("result could not be retrieved"), + failure.failure?.code === "kilo_cloud_result_unavailable" && + failure.failure.retryable === false && + failure.failure.message.includes("result could not be retrieved"), ); remote.control.missingHistory = false; const late = yield* afterRestart.readThreadSnapshot({ providerThread: afterThread }); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts index 6cb4489f5ece..f752ce7b0fc9 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -355,7 +355,8 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { status: "failed", failure: makeProviderFailure({ class: "provider_error", - code: "provider_error", + code: resultFailure ? "kilo_cloud_result_unavailable" : "provider_error", + ...(resultFailure ? { retryable: false } : {}), message: resultFailure ?? "Kilo Cloud reported a failed task.", }), threadDisposition: "reusable", @@ -1127,7 +1128,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ) ?? saved; yield* finish( saved.state, - saved.resultStatus === "unavailable" + saved.remoteState === "completed" && saved.state === "failed" ? "Kilo Cloud completed remotely, but its result was unavailable before the recovery deadline." : undefined, ); diff --git a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts index ea9c93997290..c1272162e9ca 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts @@ -33,6 +33,9 @@ const decodeOwner = Schema.decodeUnknownEffect( const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const environment = { PATH: process.env.PATH, + // Fixtures import only Node builtins; do not let optional plugin package setup + // reach the registry or inherit an external npm configuration from HOME. + npm_config_offline: "true", HTTP_PROXY: process.env.HTTP_PROXY, HTTPS_PROXY: process.env.HTTPS_PROXY, NO_PROXY: process.env.NO_PROXY, From c937f1c61af63ea2440672cb064352c00d61d7d9 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sun, 4 Oct 2026 08:04:20 +0000 Subject: [PATCH 17/44] fix(kilo): recover unsent cloud requests without retrying uncertain submissions --- .../src/components/LocalWorkspaceNotice.tsx | 32 + .../features/files/ThreadFilesRouteScreen.tsx | 37 +- .../src/features/review/ReviewSheet.tsx | 7 + .../SettingsProviderAccountsRouteScreen.tsx | 2 +- .../terminal/ThreadTerminalRouteScreen.tsx | 13 +- .../features/threads/ThreadDetailScreen.tsx | 10 +- .../src/state/threadLocalWorkspace.test.ts | 110 ++- apps/mobile/src/state/threadLocalWorkspace.ts | 22 +- .../src/state/use-selected-thread-worktree.ts | 23 +- .../Adapters/KiloCloudAdapterV2.test.ts | 670 +++++++++++++++++- .../Adapters/KiloCloudAdapterV2.ts | 149 +++- .../server/src/provider/Drivers/KiloDriver.ts | 1 + .../src/provider/kilo/KiloCloudClient.ts | 1 + .../src/provider/kilo/KiloCloudJournal.ts | 6 + .../provider/kilo/KiloCloudWebClient.test.ts | 60 +- .../src/provider/kilo/KiloCloudWebClient.ts | 74 +- .../kilo/KiloRuntime.crash.fixture.mjs | 1 + .../provider/kilo/KiloRuntime.live.test.ts | 192 ++--- apps/server/src/provider/kilo/KiloRuntime.ts | 14 +- apps/web/src/components/ChatMarkdown.tsx | 5 +- apps/web/src/components/ChatView.tsx | 10 +- docs/user/providers-kilo.md | 42 +- packages/client-runtime/package.json | 4 + .../src/cloudExecutionLabels.ts | 37 + 24 files changed, 1281 insertions(+), 241 deletions(-) create mode 100644 apps/mobile/src/components/LocalWorkspaceNotice.tsx create mode 100644 packages/client-runtime/src/cloudExecutionLabels.ts diff --git a/apps/mobile/src/components/LocalWorkspaceNotice.tsx b/apps/mobile/src/components/LocalWorkspaceNotice.tsx new file mode 100644 index 000000000000..d167ffbd34f5 --- /dev/null +++ b/apps/mobile/src/components/LocalWorkspaceNotice.tsx @@ -0,0 +1,32 @@ +import { View } from "react-native"; +import { EmptyState } from "./EmptyState"; +import { LoadingScreen } from "./LoadingScreen"; +import type { threadLocalWorkspace } from "../state/threadLocalWorkspace"; + +export function LocalWorkspaceNotice({ + state, +}: { + readonly state: ReturnType["localWorkspaceState"]; +}) { + if (state === "loading") return ; + return ( + + + + ); +} diff --git a/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx b/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx index f0ffd5de22e3..610e87f648c9 100644 --- a/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx +++ b/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx @@ -1,3 +1,4 @@ +import { LocalWorkspaceNotice } from "../../components/LocalWorkspaceNotice"; import { NativeStackScreenOptions } from "../../native/StackHeader"; import { StackActions, useNavigation, type StaticScreenProps } from "@react-navigation/native"; import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react"; @@ -351,7 +352,8 @@ function useThreadFilesWorkspace(params: { const routeCwd = firstRouteParam(params.cwd); const routeProjectName = firstRouteParam(params.projectName); const { selectedThread, selectedThreadProject } = useThreadSelection(); - const { selectedThreadCwd, localWorkspaceEnabled } = useSelectedThreadWorktree(); + const { selectedThreadCwd, localWorkspaceEnabled, localWorkspaceState } = + useSelectedThreadWorktree(); const environmentId = routeEnvironmentId !== null ? EnvironmentId.make(routeEnvironmentId) @@ -363,6 +365,7 @@ function useThreadFilesWorkspace(params: { } | null; return { + localWorkspaceState: routeThreadId !== null ? localWorkspaceState : ("unavailable" as const), cwd: routeThreadId !== null ? localWorkspaceEnabled @@ -376,18 +379,6 @@ function useThreadFilesWorkspace(params: { }; } -function FilesUnavailable() { - return ( - - - - - ); -} - function FilesToolbarBottomFade() { const sheetColor = String(useUniwindTheme()["--color-sheet"]); @@ -422,9 +413,8 @@ export function ThreadFilesTreeScreen(props: ThreadFilesRouteScreenProps) { const { fileInspector, layout, showAuxiliaryPane } = useAdaptiveWorkspaceLayout(); const [searchQuery, setSearchQuery] = useState(""); const { themeAppearance: highlightTheme } = useAppearancePreferences(); - const { cwd, environmentId, projectName, selectedThread, threadId } = useThreadFilesWorkspace( - props.route.params, - ); + const { cwd, environmentId, projectName, selectedThread, threadId, localWorkspaceState } = + useThreadFilesWorkspace(props.route.params); const revealedInspectorRef = useRef(false); const entriesQuery = useFileTreeEntries({ environmentId, @@ -502,6 +492,9 @@ export function ThreadFilesTreeScreen(props: ThreadFilesRouteScreenProps) { } }, [cwd, fileInspector.supported, showAuxiliaryPane]); + if (threadId !== null && ["cloud", "error", "unavailable"].includes(localWorkspaceState)) + return ; + if (selectedThread === null || environmentId === null || threadId === null) { if (fileInspector.supported) { return ( @@ -516,7 +509,7 @@ export function ThreadFilesTreeScreen(props: ThreadFilesRouteScreenProps) { } if (cwd === null) { - return ; + return ; } if (fileInspector.supported) { @@ -570,9 +563,8 @@ export function ThreadFileScreen(props: ThreadFileRouteScreenProps) { const params = props.route.params; const relativePath = normalizeRoutePath(params.path); const targetLine = normalizeRouteLine(firstRouteParam(params.line)); - const { cwd, environmentId, projectName, selectedThread, threadId } = useThreadFilesWorkspace( - props.route.params, - ); + const { cwd, environmentId, projectName, selectedThread, threadId, localWorkspaceState } = + useThreadFilesWorkspace(props.route.params); const [modeOverride, setModeOverride] = useState<{ readonly path: string; readonly mode: FileViewMode; @@ -863,12 +855,15 @@ export function ThreadFileScreen(props: ThreadFileRouteScreenProps) { // A file opened from a project draft has no thread, and needs none: the thread only supplies // the workspace to read from and the target to navigate back to, both of which a draft names // for itself. Wait only for what this file actually cannot render without. + if (threadId !== null && ["cloud", "error", "unavailable"].includes(localWorkspaceState)) + return ; + if (environmentId === null || (threadId !== null && selectedThread === null)) { return ; } if (cwd === null) { - return ; + return ; } if (relativePath === null) { diff --git a/apps/mobile/src/features/review/ReviewSheet.tsx b/apps/mobile/src/features/review/ReviewSheet.tsx index 20c87b496ab3..9050561475b7 100644 --- a/apps/mobile/src/features/review/ReviewSheet.tsx +++ b/apps/mobile/src/features/review/ReviewSheet.tsx @@ -1,3 +1,4 @@ +import { LocalWorkspaceNotice } from "../../components/LocalWorkspaceNotice"; import type { EnvironmentId, ThreadId } from "@t3tools/contracts"; import { useNavigation, type StaticScreenProps } from "@react-navigation/native"; import { nativeHeaderScrollEdgeEffects } from "../../native/StackHeader"; @@ -447,6 +448,12 @@ type ReviewSheetProps = StaticScreenProps<{ }>; export function ReviewSheet(props: ReviewSheetProps) { + const { localWorkspaceEnabled, localWorkspaceState } = useSelectedThreadWorktree(); + if (!localWorkspaceEnabled) return ; + return ; +} + +function LocalReviewSheet(props: ReviewSheetProps) { const { nativeReviewDiffStyle } = useAppearanceCodeSurface(); useAdaptiveWorkspacePaneRole("inspector"); const { panes, showAuxiliaryPane } = useAdaptiveWorkspaceLayout(); diff --git a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx index a07c80b4067f..b6b7b8b84cc3 100644 --- a/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsProviderAccountsRouteScreen.tsx @@ -199,7 +199,7 @@ function ProviderAccount({ } return ( - + {provider.displayName ?? provider.driver} diff --git a/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx b/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx index 2e0cfa275269..37cd74bfd9d9 100644 --- a/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx +++ b/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx @@ -1,3 +1,4 @@ +import { LocalWorkspaceNotice } from "../../components/LocalWorkspaceNotice"; import { useSelectedThreadWorktree } from "../../state/use-selected-thread-worktree"; import { DEFAULT_TERMINAL_ID, EnvironmentId, ThreadId } from "@t3tools/contracts"; import { type KnownTerminalSession } from "@t3tools/client-runtime/state/terminal"; @@ -241,16 +242,8 @@ type ThreadTerminalRouteScreenProps = StaticScreenProps<{ }>; export function ThreadTerminalRouteScreen(props: ThreadTerminalRouteScreenProps) { - const { localWorkspaceEnabled } = useSelectedThreadWorktree(); - if (!localWorkspaceEnabled) - return ( - - - - ); + const { localWorkspaceEnabled, localWorkspaceState } = useSelectedThreadWorktree(); + if (!localWorkspaceEnabled) return ; return ; } diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index 9b8543a2623f..803f3a5487c8 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -1,3 +1,4 @@ +import { cloudExecutionLabel } from "@t3tools/client-runtime/cloudExecutionLabels"; import { AppText } from "../../components/AppText"; import { useThreadCloudExecution, useThreadReportedModelSelection } from "../../state/entities"; import { UsageLimitRecoveryCard } from "./UsageLimitRecoveryCard"; @@ -1276,9 +1277,12 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread Last observation: {cloudExecution.observedAt ?? "unavailable"}. Task:{" "} - {cloudExecution.task}.{" "} - {cloudExecution.result ? `Result: ${cloudExecution.result}. ` : ""}Sandbox:{" "} - {cloudExecution.sandbox}. Compute: {cloudExecution.billing} + {cloudExecutionLabel(cloudExecution.task)}.{" "} + {cloudExecution.result + ? `Result: ${cloudExecutionLabel(cloudExecution.result)}. ` + : ""} + Sandbox: {cloudExecutionLabel(cloudExecution.sandbox)}. Compute:{" "} + {cloudExecutionLabel(cloudExecution.billing)} {cloudExecution.billingAttribution === "payer_shared" ? " (shared account)" : ""} diff --git a/apps/mobile/src/state/threadLocalWorkspace.test.ts b/apps/mobile/src/state/threadLocalWorkspace.test.ts index 71decf636cb9..8e72f532c1fa 100644 --- a/apps/mobile/src/state/threadLocalWorkspace.test.ts +++ b/apps/mobile/src/state/threadLocalWorkspace.test.ts @@ -1,21 +1,53 @@ import { describe, expect, it } from "vite-plus/test"; +import * as DateTime from "effect/DateTime"; import { ProviderDriverKind, ProviderThreadId, - type OrchestrationV2ThreadProjection, + ProviderInstanceId, + type OrchestrationV2ProviderThread, } from "@t3tools/contracts"; import { threadLocalWorkspace } from "./threadLocalWorkspace"; +const providerThread: OrchestrationV2ProviderThread = { + id: ProviderThreadId.make("native-local"), + driver: ProviderDriverKind.make("kilo"), + providerInstanceId: ProviderInstanceId.make("kilo-personal"), + providerSessionId: null, + appThreadId: null, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: DateTime.makeUnsafe(0), + updatedAt: DateTime.makeUnsafe(0), +}; +const local = { + driver: providerThread.driver, + providerThreads: [providerThread], + activeProviderThreadId: providerThread.id, + worktreePath: "/local/worktree", + workspaceRoot: "/local/repository", + detailLoaded: true, + providerConfigLoaded: true, +}; +function expectBlocked(input: Parameters[0], state: string) { + expect(threadLocalWorkspace(input)).toEqual({ + localWorkspaceState: state, + localWorkspaceEnabled: false, + selectedThreadWorktreePath: null, + selectedThreadCwd: null, + selectedThreadGitRootCwd: null, + }); +} describe("mobile thread workspace routing", () => { - const local = { - driver: ProviderDriverKind.make("codex"), - providerThreads: [], - activeProviderThreadId: null, - worktreePath: "/local/worktree", - workspaceRoot: "/local/repository", - }; - it("keeps local worktree and repository actions available for a known local provider", () => { + it("restores actual local Kilo worktree, repository and draft flows after hydration", () => { + expectBlocked({ ...local, providerThreads: [], detailLoaded: false }, "loading"); expect(threadLocalWorkspace(local)).toEqual({ + localWorkspaceState: "local", localWorkspaceEnabled: true, selectedThreadWorktreePath: "/local/worktree", selectedThreadCwd: "/local/worktree", @@ -24,33 +56,57 @@ describe("mobile thread workspace routing", () => { expect(threadLocalWorkspace({ ...local, worktreePath: null }).selectedThreadCwd).toBe( "/local/repository", ); + expect( + threadLocalWorkspace({ ...local, activeProviderThreadId: null, providerThreads: [] }) + .localWorkspaceEnabled, + ).toBe(true); + expectBlocked({ ...local, worktreePath: null, workspaceRoot: null }, "unavailable"); + }); + it("distinguishes config loading, failed reads and a removed provider without opening local resources", () => { + expectBlocked({ ...local, driver: undefined, providerConfigLoaded: false }, "loading"); + expectBlocked( + { ...local, driver: undefined, providerConfigLoaded: false, loadError: "Disconnected" }, + "error", + ); + expectBlocked({ ...local, detailLoaded: false, loadError: "Detail failed" }, "error"); + expectBlocked({ ...local, driver: undefined }, "unavailable"); + expectBlocked({ ...local, providerThreads: [] }, "unavailable"); + expectBlocked({ ...local, detailLoaded: false, threadDeleted: true }, "unavailable"); + expect(threadLocalWorkspace({ ...local, loadError: null }).localWorkspaceEnabled).toBe(true); }); - it("never supplies a local Git/file target for cloud, missing providers or persisted cloud history", () => { - const persisted = [ - { id: "remote", nativeMetadata: { cloudExecution: { sessionId: "workspace_remote" } } }, - ] as unknown as OrchestrationV2ThreadProjection["providerThreads"]; + it("keeps known cloud metadata authoritative during config loading and failure", () => { for (const input of [ { ...local, driver: ProviderDriverKind.make("kilo-cloud") }, - { ...local, driver: undefined }, - { ...local, activeProviderThreadId: ProviderThreadId.make("not-loaded") }, { ...local, - activeProviderThreadId: ProviderThreadId.make("cloud-before-admission"), - providerThreads: [ - { id: "cloud-before-admission", driver: "kilo-cloud" }, - ] as unknown as OrchestrationV2ThreadProjection["providerThreads"], + driver: undefined, + providerConfigLoaded: false, + providerThreads: [{ ...providerThread, driver: ProviderDriverKind.make("kilo-cloud") }], }, { ...local, - providerThreads: persisted, - activeProviderThreadId: ProviderThreadId.make("remote"), + loadError: "Disconnected", + providerThreads: [ + { + ...providerThread, + nativeMetadata: { + cloudExecution: { + repository: "fixture/repo", + branch: "main", + sessionId: null, + worktreeId: null, + task: "not_started" as const, + sandbox: "unknown" as const, + billing: "unknown" as const, + billingAttribution: null, + estimatedHourlyRateUsd: null, + observedAt: null, + }, + }, + }, + ], }, ]) - expect(threadLocalWorkspace(input)).toEqual({ - localWorkspaceEnabled: false, - selectedThreadWorktreePath: null, - selectedThreadCwd: null, - selectedThreadGitRootCwd: null, - }); + expectBlocked(input, "cloud"); }); }); diff --git a/apps/mobile/src/state/threadLocalWorkspace.ts b/apps/mobile/src/state/threadLocalWorkspace.ts index ea5c3105aae2..60d50245cf56 100644 --- a/apps/mobile/src/state/threadLocalWorkspace.ts +++ b/apps/mobile/src/state/threadLocalWorkspace.ts @@ -5,6 +5,10 @@ export function threadLocalWorkspace(input: { readonly driver: ProviderDriverKind | undefined; readonly providerThreads: OrchestrationV2ThreadProjection["providerThreads"]; readonly activeProviderThreadId: OrchestrationV2ThreadProjection["thread"]["activeProviderThreadId"]; + readonly detailLoaded?: boolean; + readonly threadDeleted?: boolean; + readonly providerConfigLoaded?: boolean; + readonly loadError?: string | null; readonly worktreePath: string | null; readonly workspaceRoot: string | null; }) { @@ -14,8 +18,24 @@ export function threadLocalWorkspace(input: { active?.driver === "kilo-cloud" || !!active?.nativeMetadata?.cloudExecution; const resolved = input.activeProviderThreadId === null || active !== undefined; - const enabled = resolved && !cloud && input.driver !== undefined; + const state = cloud + ? "cloud" + : input.threadDeleted + ? "unavailable" + : input.loadError + ? "error" + : input.detailLoaded === false || input.providerConfigLoaded === false + ? "loading" + : !resolved || input.driver === undefined + ? input.providerConfigLoaded === true && input.detailLoaded === true + ? "unavailable" + : "loading" + : !(input.worktreePath ?? input.workspaceRoot) + ? "unavailable" + : "local"; + const enabled = state === "local"; return { + localWorkspaceState: state, localWorkspaceEnabled: enabled, selectedThreadWorktreePath: enabled ? input.worktreePath : null, selectedThreadCwd: enabled ? (input.worktreePath ?? input.workspaceRoot) : null, diff --git a/apps/mobile/src/state/use-selected-thread-worktree.ts b/apps/mobile/src/state/use-selected-thread-worktree.ts index 30663376eaca..811367dde508 100644 --- a/apps/mobile/src/state/use-selected-thread-worktree.ts +++ b/apps/mobile/src/state/use-selected-thread-worktree.ts @@ -5,12 +5,23 @@ import { useSelectedThreadWorktreePath, useSelectedThreadDetailState } from "./u import { useThreadSelection } from "./use-thread-selection"; import { resolvePreferredThreadWorktreePath } from "../features/terminal/terminalLaunchContext"; +import { useEnvironmentQuery } from "./query"; +import { serverEnvironment } from "./server"; import { threadLocalWorkspace } from "./threadLocalWorkspace"; export function useSelectedThreadWorktree() { const { selectedThread, selectedThreadProject, selectedEnvironmentRuntime } = useThreadSelection(); - const projection = Option.getOrNull(useSelectedThreadDetailState().data); + const detail = useSelectedThreadDetailState(); + const projection = Option.getOrNull(detail.data); + const config = useEnvironmentQuery( + selectedThread?.environmentId + ? serverEnvironment.configProjection({ + environmentId: selectedThread.environmentId, + input: {}, + }) + : null, + ); const detailWorktreePath = useSelectedThreadWorktreePath(); const selectedThreadWorktreePath = useMemo( @@ -23,9 +34,17 @@ export function useSelectedThreadWorktree() { ); return threadLocalWorkspace({ - driver: selectedEnvironmentRuntime?.serverConfig?.providers.find( + driver: config.data?.config.providers.find( (provider) => provider.instanceId === selectedThread?.providerInstanceId, )?.driver, + detailLoaded: projection !== null, + threadDeleted: detail.status === "deleted", + providerConfigLoaded: config.data !== null, + loadError: + Option.getOrNull(detail.error) ?? + config.error ?? + selectedEnvironmentRuntime?.connectionError ?? + null, providerThreads: projection?.providerThreads ?? [], activeProviderThreadId: selectedThread?.activeProviderThreadId ?? null, worktreePath: selectedThreadWorktreePath, diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index 0d9d98c4cc8c..a702928b36b4 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -3,6 +3,12 @@ import * as NodeChildProcess from "node:child_process"; import * as KiloRuntime from "../../provider/kilo/KiloRuntime.ts"; import * as KiloAdapter from "./KiloAdapterV2.ts"; import * as NodeHttp from "node:http"; +import * as NodeFS from "node:fs"; +import * as Account from "../../provider/kilo/KiloCloudAccount.ts"; +import * as DateTime from "effect/DateTime"; +import * as Clock from "effect/Clock"; +import * as TestClock from "effect/testing/TestClock"; +import { NodeId, ProviderDriverKind } from "@t3tools/contracts"; import * as NodeEvents from "node:events"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; @@ -38,6 +44,14 @@ import { makeOrchestratorV2ReplayLayerWithRegistry } from "../testkit/ProviderRe import type * as Adapter from "../ProviderAdapter.ts"; import * as CloudAdapter from "./KiloCloudAdapterV2.ts"; +const clockAt = (clock: Clock.Clock, millis: number): Clock.Clock => ({ + ...clock, + currentTimeMillis: Effect.succeed(millis), + currentTimeMillisUnsafe: () => millis, + currentTimeNanos: Effect.succeed(BigInt(millis) * 1_000_000n), + currentTimeNanosUnsafe: () => BigInt(millis) * 1_000_000n, + sleep: clock.sleep.bind(clock), +}); const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); const fixture = Effect.acquireRelease( Effect.promise(async () => { @@ -46,7 +60,13 @@ const fixture = Effect.acquireRelease( let localRequests = 0; let overlapped = false; const completeLocal = () => { - if (!localResponse || submissions < 2) return; + if ( + !localResponse || + localResponse.destroyed || + localResponse.writableEnded || + submissions < 2 + ) + return; overlapped = true; localResponse.writeHead(200, { "content-type": "text/event-stream" }); for (const choice of [ @@ -64,9 +84,29 @@ const fixture = Effect.acquireRelease( signalInterrupt = resolve; }); const control = { + listStatus: 200, + parkList: false, + listSeen: undefined as (() => void) | undefined, + listClosed: undefined as (() => void) | undefined, + resultReads: 0, + completeAfterResultReads: 0, + sessionStatus: 200, + malformedSession: false, + sendPosts: 0, + profileStatus: 200, + personalAccount: true, + profileAccount: "fixture-account", + malformedProfile: false, + preflightStatus: 200, + malformedPreflight: false, + afterBindings: undefined as (() => void) | undefined, + parkedPreflight: undefined as NodeHttp.ServerResponse | undefined, + parkPreflight: false, + preflightSeen: undefined as (() => void) | undefined, status: "completed", requireLocalOverlap: false, dropNextPrepare: false, + hideAdmissions: false, omittedItemCount: 0, missingHistory: false, incompleteHistory: false, @@ -101,6 +141,12 @@ const fixture = Effect.acquireRelease( if (url.pathname.endsWith("/chat/completions")) { localRequests++; localResponse = response; + response.once("close", () => { + if (localResponse === response) localResponse = undefined; + }); + response.on("error", () => { + if (localResponse === response) localResponse = undefined; + }); completeLocal(); return; } @@ -113,7 +159,37 @@ const fixture = Effect.acquireRelease( response.writeHead(200, { "content-type": "application/json" }); response.end(JSON.stringify({ result: { data } })); }; - if (operation.startsWith("agentProfiles.")) return reply([]); + if (url.pathname === "/api/profile") { + response.writeHead(control.profileStatus, { "content-type": "application/json" }); + response.end( + JSON.stringify( + control.malformedProfile + ? {} + : { + user: { id: control.profileAccount }, + hasPersonalAccount: control.personalAccount, + }, + ), + ); + return; + } + if (operation.startsWith("agentProfiles.")) { + if (control.parkPreflight) { + control.parkedPreflight = response; + response.once("close", () => { + if (control.parkedPreflight === response) control.parkedPreflight = undefined; + }); + control.preflightSeen?.(); + return; + } + if (control.preflightStatus !== 200) { + response.writeHead(control.preflightStatus); + response.end(); + return; + } + if (operation.endsWith("listRepoBindings")) control.afterBindings?.(); + return reply(control.malformedPreflight ? {} : []); + } if (operation === "cloudAgentNext.prepareSession") { submissions++; completeLocal(); @@ -133,12 +209,25 @@ const fixture = Effect.acquireRelease( } return reply({ cloudAgentSessionId: state.cloud, kiloSessionId: state.native }); } + if (operation === "cliSessionsV2.list" && control.parkList) { + response.once("close", () => control.listClosed?.()); + response.on("error", () => {}); + control.listSeen?.(); + return; + } + if (operation === "cliSessionsV2.list" && control.listStatus !== 200) { + response.writeHead(control.listStatus); + response.end(); + return; + } if (operation === "cliSessionsV2.list") return reply({ - cliSessions: [...conversations.values()].map((state) => ({ - session_id: state.native, - cloud_agent_session_id: state.cloud, - })), + cliSessions: (control.hideAdmissions ? [] : [...conversations.values()]).map( + (state) => ({ + session_id: state.native, + cloud_agent_session_id: state.cloud, + }), + ), nextCursor: null, }); const state = [...conversations.values()].find( @@ -166,6 +255,12 @@ const fixture = Effect.acquireRelease( response.end(); return; } + if (operation === "cloudAgentNext.getSession" && control.sessionStatus !== 200) { + response.writeHead(control.sessionStatus); + response.end(); + return; + } + if (operation === "cloudAgentNext.getSession" && control.malformedSession) return reply({}); if (operation === "cloudAgentNext.getSession") return reply({ sessionId: state.cloud, @@ -179,6 +274,7 @@ const fixture = Effect.acquireRelease( execution: null, }); if (operation === "cloudAgentNext.sendMessage") { + control.sendPosts++; const payload = input.payload as unknown as { prompt: string }; state.messages.push({ id: input.messageId!, prompt: payload.prompt }); return reply({ @@ -235,19 +331,27 @@ const fixture = Effect.acquireRelease( estimatedHourlyRateMicrodollars: 0, estimatedIntervalAmountMicrodollars: 0, }); + if (operation === "cloudAgentNext.getMessageResult") control.resultReads++; if (operation === "cloudAgentNext.getMessageResult") return reply({ cloudAgentSessionId: state.cloud, messageId: input.messageId, - status: control.requireLocalOverlap && !localRequests ? "running" : control.status, + status: + control.completeAfterResultReads > 0 + ? control.resultReads >= control.completeAfterResultReads + ? "completed" + : "running" + : control.requireLocalOverlap && !localRequests + ? "running" + : control.status, }); + if (operation === "cliSessionsV2.getSessionMessagesPage") control.historyReads++; if (operation === "cliSessionsV2.getSessionMessagesPage" && control.missingHistory) return reply({ kiloSessionId: state.native, history: null, watermarkEventId: 49 }); if ( operation === "cliSessionsV2.getSessionMessagesPage" && control.historyMode !== "normal" ) { - control.historyReads++; const message = control.historyMode === "older" && input.cursor === "1" ? state.messages[0]! @@ -850,8 +954,8 @@ it.live( const awaiting = yield* Deferred.make(); yield* retrieving.events.pipe( Stream.runForEach((event) => - event.type === "provider_session.updated" && - event.providerSession.lastError?.includes("Awaiting its correlated result") + event.type === "provider_thread.updated" && + event.providerThread.nativeMetadata?.cloudExecution?.result === "awaiting_result" ? Deferred.succeed(awaiting, undefined) : Effect.void, ), @@ -955,8 +1059,8 @@ it.live( const cancelAwaiting = yield* Deferred.make(); yield* cancelling.events.pipe( Stream.runForEach((event) => - event.type === "provider_session.updated" && - event.providerSession.lastError?.includes("Awaiting its correlated result") + event.type === "provider_thread.updated" && + event.providerThread.nativeMetadata?.cloudExecution?.result === "awaiting_result" ? Deferred.succeed(cancelAwaiting, undefined) : Effect.void, ), @@ -1014,8 +1118,8 @@ it.live( Effect.gen(function* () { if (event.type === "turn.terminal") yield* Deferred.succeed(done, event); if ( - event.type === "provider_session.updated" && - event.providerSession.lastError?.includes("Awaiting its correlated result") + event.type === "provider_thread.updated" && + event.providerThread.nativeMetadata?.cloudExecution?.result === "awaiting_result" ) yield* Deferred.succeed(waitingResult, undefined); if ( @@ -1044,6 +1148,10 @@ it.live( providerThread: selected, providerTurnId: unfinished.providerTurn.id, }); + const stopped = (yield* journal.read).at(-1)!; + assert.equal(stopped.state, "interrupted"); + assert.equal(stopped.resultStatus, "cancelled"); + assert.equal(stopped.remoteState, "completed"); } else { if (mode === "paged") { yield* Deferred.await(waitingResult); @@ -1102,3 +1210,537 @@ it.live( }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), 60_000, ); + +const admissionHarness = Effect.fn("admissionHarness")(function* ( + remote: Effect.Success, + directory: string, +) { + const instanceId = ProviderInstanceId.make("cloud-admission"); + const threadId = ThreadId.make("admission-thread"); + const now = yield* DateTime.now; + const modelSelection = { instanceId, model: "fixture/model" }; + const runtimePolicy = { + runtimeMode: "full-access" as const, + interactionMode: "default" as const, + cwd: null, + }; + const account = yield* Account.make(directory, remote.origin); + const journal = yield* Journal.make(`${directory}/journal`); + const client = Cloud.make({ + accountId: "fixture-account", + token: Redacted.make("fixture"), + origin: remote.origin, + credentials: account.load, + }); + const adapter = yield* CloudAdapter.make({ + instanceId, + continuationKey: "admission-account", + accountId: "fixture-account", + repository: "fixture/repo", + branch: "main", + client, + journal, + }); + const initial: import("@t3tools/contracts").OrchestrationV2ProviderThread = { + id: ProviderThreadId.make("admission-provider-thread"), + driver: ProviderDriverKind.make("kilo-cloud"), + providerInstanceId: instanceId, + providerSessionId: null, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }; + const open = Effect.gen(function* () { + const runtime = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("admission-session"), + modelSelection, + runtimePolicy, + }); + const thread = yield* runtime.resumeThread({ providerThread: initial }); + return { runtime, thread }; + }); + const turn = (thread: typeof initial, ordinal = 1): Adapter.ProviderAdapterV2TurnInput => ({ + appThread: { + id: threadId, + projectId: ProjectId.make("admission-project"), + title: "Admission fixture", + providerInstanceId: instanceId, + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: thread.id, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, + forkedFrom: null, + createdBy: "user", + creationSource: "web", + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + lastVisitedAt: null, + deletedAt: null, + }, + threadId, + runId: RunId.make(`admission-run-${ordinal}`), + attemptId: RunAttemptId.make(`admission-attempt-${ordinal}`), + rootNodeId: NodeId.make(`admission-node-${ordinal}`), + runOrdinal: ordinal, + providerTurnOrdinal: ordinal, + providerThread: thread, + message: { + messageId: MessageId.make(`admission-message-${ordinal}`), + text: "Read synthetic README", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection, + runtimePolicy, + }); + return { open, turn, journal, client }; +}); + +it.live.each([ + "404", + "503", + "malformed", + "profile-503", + "profile-malformed", + "personal-account", + "wrong-account", + "credential", + "last-credential", +] as const)( + "ends proven unsent %s preflight and permits an explicit next turn after restart", + (mode) => + Effect.gen(function* () { + const remote = yield* fixture; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + const auth = `${directory}/data/kilo/auth.json`; + const validAuth = '{"kilo":{"type":"api","key":"synthetic-test-token"}}'; + yield* fs.writeFileString(auth, mode === "credential" ? "{}" : validAuth); + if (mode === "404" || mode === "503") remote.control.preflightStatus = Number(mode); + if (mode === "malformed") remote.control.malformedPreflight = true; + if (mode === "profile-503") remote.control.profileStatus = 503; + if (mode === "profile-malformed") remote.control.malformedProfile = true; + if (mode === "personal-account") remote.control.personalAccount = false; + if (mode === "wrong-account") remote.control.profileAccount = "another-account"; + if (mode === "last-credential") + remote.control.afterBindings = () => NodeFS.writeFileSync(auth, "{}"); + const harness = yield* admissionHarness(remote, directory); + const scope = yield* Scope.fork(yield* Effect.scope); + const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, scope)); + yield* first.runtime.startTurn(harness.turn(first.thread)); + const intent = (yield* harness.journal.read)[0]!; + assert.equal(remote.submissions(), 0); + assert.equal(intent.state, "failed"); + assert.equal(intent.submissionPhase, "preflight"); + yield* first.runtime.interruptTurn({ + providerThread: first.thread, + providerTurnId: intent.providerTurn.id, + }); + assert.equal(remote.control.interruptPosts, 0); + yield* Scope.close(scope, Exit.void); + remote.control.preflightStatus = 200; + remote.control.malformedPreflight = false; + remote.control.profileStatus = 200; + remote.control.malformedProfile = false; + remote.control.personalAccount = true; + remote.control.profileAccount = "fixture-account"; + remote.control.afterBindings = undefined; + yield* fs.writeFileString(auth, validAuth); + // New account, client, journal and runtime; the original durable record remains. + const restarted = yield* admissionHarness(remote, directory); + const next = yield* restarted.open; + remote.control.status = "failed"; + const done = yield* Deferred.make(); + yield* next.runtime.events.pipe( + Stream.runForEach((event) => + event.type === "turn.terminal" ? Deferred.succeed(done, undefined) : Effect.void, + ), + Effect.forkScoped, + ); + yield* next.runtime.startTurn(restarted.turn(next.thread, 2)); + yield* Deferred.await(done); + assert.equal(remote.submissions(), 1); + assert.equal((yield* restarted.journal.read)[1]?.submissionPhase, "post_attempted"); + assert.equal((yield* restarted.journal.read)[1]?.state, "failed"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live.each([false, true])( + "keeps a lost POST uncertain across restart and Stop, legacy=%s", + (legacy) => + Effect.gen(function* () { + const remote = yield* fixture; + remote.control.dropNextPrepare = true; + remote.control.hideAdmissions = true; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const harness = yield* admissionHarness(remote, directory); + const scope = yield* Scope.fork(yield* Effect.scope); + const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, scope)); + yield* first.runtime.startTurn(harness.turn(first.thread)); + yield* Scope.close(scope, Exit.void); + let uncertain = (yield* harness.journal.read)[0]!; + assert.equal(uncertain.state, "admission_unknown"); + assert.equal(uncertain.submissionPhase, "post_attempted"); + assert.equal(remote.submissions(), 1); + if (legacy) { + const old = { ...uncertain }; + delete old.submissionPhase; + uncertain = yield* harness.journal.save(old); + } + const restarted = yield* admissionHarness(remote, directory); + const next = yield* restarted.open; + yield* next.runtime.startTurn({ ...restarted.turn(next.thread), reattach: true }); + const baseClock = yield* Clock.Clock; + const laterClock = clockAt(baseClock, (yield* Clock.currentTimeMillis) + 86_400_000); + yield* next.runtime + .readThreadSnapshot({ providerThread: next.thread }) + .pipe(Effect.provideService(Clock.Clock, laterClock)); + const stop = yield* next.runtime + .interruptTurn({ providerThread: next.thread, providerTurnId: uncertain.providerTurn.id }) + .pipe(Effect.flip); + assert.include(stop.message, "no confirmed session ID"); + const retry = yield* next.runtime.startTurn(restarted.turn(next.thread, 2)).pipe(Effect.flip); + assert.include(retry.message, "admission is unknown"); + assert.equal((yield* restarted.journal.read)[0]?.state, "admission_unknown"); + assert.equal(remote.submissions(), 1); + assert.equal(remote.control.interruptPosts, 0); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live( + "a recovered preflight reservation prevents the original waiting adapter from dispatching", + () => + Effect.gen(function* () { + const remote = yield* fixture; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const arrived = yield* Deferred.make(); + remote.control.parkPreflight = true; + remote.control.preflightSeen = () => Deferred.doneUnsafe(arrived, Effect.void); + const firstHarness = yield* admissionHarness(remote, directory); + const first = yield* firstHarness.open; + const pending = yield* first.runtime + .startTurn(firstHarness.turn(first.thread)) + .pipe(Effect.forkScoped); + yield* Deferred.await(arrived); + assert.equal((yield* firstHarness.journal.read)[0]?.submissionPhase, "preflight"); + const recovery = yield* admissionHarness(remote, directory); + const reopened = yield* recovery.open; + const recovered = (yield* recovery.journal.read)[0]!; + assert.equal(recovered.state, "failed"); + yield* reopened.runtime.interruptTurn({ + providerThread: reopened.thread, + providerTurnId: recovered.providerTurn.id, + }); + remote.control.parkPreflight = false; + remote.control.parkedPreflight!.writeHead(200, { "content-type": "application/json" }); + remote.control.parkedPreflight!.end('{"result":{"data":[]}}'); + const originalExit = yield* Fiber.await(pending); + assert.isTrue(Exit.isFailure(originalExit)); // stale CAS cannot revive the reservation + assert.equal(remote.submissions(), 0); + assert.equal((yield* recovery.journal.read)[0]?.state, "failed"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live( + "persists result backoff and reattaches without resetting attempts or the deadline", + () => + Effect.gen(function* () { + const remote = yield* fixture; + remote.control.missingHistory = true; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const harness = yield* admissionHarness(remote, directory); + const firstScope = yield* Scope.fork(yield* Effect.scope); + const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, firstScope)); + const waiting = yield* Deferred.make(); + yield* first.runtime.events.pipe( + Stream.runForEach((event) => + event.type === "provider_thread.updated" && + event.providerThread.nativeMetadata?.cloudExecution?.result === "awaiting_result" + ? Deferred.succeed(waiting, undefined) + : Effect.void, + ), + Effect.forkIn(firstScope), + ); + yield* first.runtime.startTurn(harness.turn(first.thread)); + yield* Deferred.await(waiting); + yield* Scope.close(firstScope, Exit.void); + const saved = (yield* harness.journal.read)[0]!; + assert.equal(saved.state, "awaiting_result"); + assert.equal(saved.resultRecovery?.attempts, 1); + const deadline = saved.resultRecovery!.deadlineMs; + const due = saved.resultRecovery!.nextAttemptMs; + const baseClock = yield* Clock.Clock; + const restarted = yield* admissionHarness(remote, directory); + const next = yield* restarted.open; + const reads = remote.control.historyReads; + yield* next.runtime + .readThreadSnapshot({ providerThread: next.thread }) + .pipe(Effect.provideService(Clock.Clock, clockAt(baseClock, due - 1))); + assert.equal(remote.control.historyReads, reads); + assert.deepEqual((yield* restarted.journal.read)[0]?.resultRecovery, saved.resultRecovery); + yield* next.runtime + .readThreadSnapshot({ providerThread: next.thread }) + .pipe(Effect.provideService(Clock.Clock, clockAt(baseClock, due))); + const second = (yield* restarted.journal.read)[0]!.resultRecovery!; + assert.equal(second.attempts, 2); + assert.equal(second.nextAttemptMs, due + 4_000); + assert.equal(second.deadlineMs, deadline); + assert.equal(remote.control.historyReads, reads + 1); + let recovery = second; + for (const delay of [8_000, 16_000, 30_000, 30_000]) { + const at = recovery.nextAttemptMs; + yield* next.runtime + .readThreadSnapshot({ providerThread: next.thread }) + .pipe(Effect.provideService(Clock.Clock, clockAt(baseClock, at))); + const following = (yield* restarted.journal.read)[0]!.resultRecovery!; + assert.equal(following.attempts, recovery.attempts + 1); + assert.equal(following.nextAttemptMs, at + delay); + assert.equal(following.deadlineMs, deadline); + recovery = following; + } + + // Reattach the original turn at expiry. Its terminal event must be replayable, + // with remote completion retained and no replacement paid submission. + const terminal = yield* Deferred.make(); + yield* next.runtime.events.pipe( + Stream.runForEach((event) => + event.type === "turn.terminal" ? Deferred.succeed(terminal, event) : Effect.void, + ), + Effect.forkScoped, + ); + yield* next.runtime + .startTurn({ ...restarted.turn(next.thread), reattach: true }) + .pipe(Effect.provideService(Clock.Clock, clockAt(baseClock, deadline + 1))); + const failed = yield* Deferred.await(terminal); + assert.isTrue(failed.type === "turn.terminal" && failed.status === "failed"); + const ended = (yield* restarted.journal.read)[0]!; + assert.equal(ended.state, "failed"); + assert.equal(ended.remoteState, "completed"); + assert.equal(ended.resultStatus, "unavailable"); + assert.equal(remote.submissions(), 1); + const replay = yield* restarted.open; + const replayed = yield* Deferred.make(); + yield* replay.runtime.events.pipe( + Stream.runForEach((event) => + event.type === "turn.terminal" ? Deferred.succeed(replayed, event) : Effect.void, + ), + Effect.forkScoped, + ); + yield* replay.runtime.startTurn({ ...restarted.turn(replay.thread), reattach: true }); + const event = yield* Deferred.await(replayed); + assert.isTrue(event.type === "turn.terminal" && event.status === "failed"); + if (event.type === "turn.terminal" && event.status === "failed") + assert.include(event.failure?.message ?? "", "unavailable before the recovery deadline"); + assert.equal(remote.submissions(), 1); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live.each(["404", "503", "malformed", "credential"] as const)( + "does not dispatch a follow-up after %s preflight failure and keeps the remote binding", + (mode) => + Effect.gen(function* () { + const remote = yield* fixture; + remote.control.status = "failed"; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + const auth = `${directory}/data/kilo/auth.json`; + const validAuth = '{"kilo":{"type":"api","key":"synthetic"}}'; + yield* fs.writeFileString(auth, validAuth); + const harness = yield* admissionHarness(remote, directory); + const firstScope = yield* Scope.fork(yield* Effect.scope); + const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, firstScope)); + const done = yield* Deferred.make(); + yield* first.runtime.events.pipe( + Stream.runForEach((event) => + event.type === "turn.terminal" ? Deferred.succeed(done, undefined) : Effect.void, + ), + Effect.forkIn(firstScope), + ); + yield* first.runtime.startTurn(harness.turn(first.thread)); + yield* Deferred.await(done); + yield* Scope.close(firstScope, Exit.void); + const binding = (yield* harness.journal.read)[0]!.binding; + assert.isNotNull(binding); + if (mode === "404" || mode === "503") remote.control.sessionStatus = Number(mode); + if (mode === "malformed") remote.control.malformedSession = true; + if (mode === "credential") yield* fs.writeFileString(auth, "{}"); + const secondScope = yield* Scope.fork(yield* Effect.scope); + const second = yield* harness.open.pipe(Effect.provideService(Scope.Scope, secondScope)); + yield* second.runtime.startTurn(harness.turn(second.thread, 2)); + const failed = (yield* harness.journal.read)[1]!; + assert.equal(failed.state, "failed"); + assert.equal(failed.submissionPhase, "preflight"); + assert.deepEqual(failed.binding, binding); + assert.equal(remote.control.sendPosts, 0); + yield* second.runtime.interruptTurn({ + providerThread: second.thread, + providerTurnId: failed.providerTurn.id, + }); + assert.equal(remote.control.interruptPosts, 0); + yield* Scope.close(secondScope, Exit.void); + remote.control.sessionStatus = 200; + remote.control.malformedSession = false; + yield* fs.writeFileString(auth, validAuth); + const restarted = yield* admissionHarness(remote, directory); + const third = yield* restarted.open; + const completed = yield* Deferred.make(); + yield* third.runtime.events.pipe( + Stream.runForEach((event) => + event.type === "turn.terminal" ? Deferred.succeed(completed, undefined) : Effect.void, + ), + Effect.forkScoped, + ); + yield* third.runtime.startTurn(restarted.turn(third.thread, 3)); + yield* Deferred.await(completed); + assert.equal(remote.submissions(), 1); + assert.equal(remote.control.sendPosts, 1); + assert.equal((yield* restarted.journal.read)[2]?.submissionPhase, "post_attempted"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live( + "pauses incomplete admission scans durably and restarts observation after a manual read retry", + () => + Effect.gen(function* () { + const remote = yield* fixture; + remote.control.dropNextPrepare = true; + remote.control.listStatus = 503; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const harness = yield* admissionHarness(remote, directory); + const scope = yield* Scope.fork(yield* Effect.scope); + const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, scope)); + yield* first.runtime.startTurn(harness.turn(first.thread)); + yield* Scope.close(scope, Exit.void); + const restarted = yield* admissionHarness(remote, directory); + const next = yield* restarted.open; + const baseClock = yield* Clock.Clock; + const now = yield* Clock.currentTimeMillis; + for (let i = 0; i < 3; i++) { + if ((yield* restarted.journal.read)[0]?.admissionRecoveryPaused) break; + yield* next.runtime + .readThreadSnapshot({ providerThread: next.thread }) + .pipe(Effect.provideService(Clock.Clock, clockAt(baseClock, now + i * 61_000))); + } + const paused = (yield* restarted.journal.read)[0]!; + assert.equal(paused.state, "admission_unknown"); + assert.isTrue(paused.admissionRecoveryPaused); + assert.equal(paused.admissionRecoveryFailures, 3); + assert.isFalse(yield* next.runtime.hasPendingBackgroundWork!); + const again = yield* admissionHarness(remote, directory); + const recovered = yield* again.open; + yield* recovered.runtime.startTurn({ ...again.turn(recovered.thread), reattach: true }); + assert.isTrue((yield* again.journal.read)[0]?.admissionRecoveryPaused); + assert.isFalse(yield* recovered.runtime.hasPendingBackgroundWork!); + remote.control.listStatus = 200; + remote.control.resultReads = 0; + remote.control.completeAfterResultReads = 2; + const done = yield* Deferred.make(); + yield* recovered.runtime.events.pipe( + Stream.runForEach((event) => + event.type === "turn.terminal" ? Deferred.succeed(done, undefined) : Effect.void, + ), + Effect.forkScoped, + ); + yield* recovered.runtime.readThreadSnapshot({ providerThread: recovered.thread }); + yield* Deferred.await(done); // watcher must finish without a second snapshot request + const complete = (yield* again.journal.read)[0]!; + assert.equal(complete.state, "completed"); + assert.isFalse(complete.admissionRecoveryPaused); + assert.isAtLeast(remote.control.resultReads, 2); + assert.equal(remote.submissions(), 1); + assert.equal(remote.control.sendPosts, 0); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.effect( + "bounds stalled admission-list reads across durable failures without releasing a paid intent", + () => + Effect.gen(function* () { + const remote = yield* fixture; + remote.control.dropNextPrepare = true; + remote.control.hideAdmissions = true; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const harness = yield* admissionHarness(remote, directory); + const firstScope = yield* Scope.fork(yield* Effect.scope); + const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, firstScope)); + yield* first.runtime.startTurn(harness.turn(first.thread)); + yield* Scope.close(firstScope, Exit.void); + const restarted = yield* admissionHarness(remote, directory); + const next = yield* restarted.open; + remote.control.parkList = true; + for (let attempt = 1; attempt <= 3; attempt++) { + const requestSeen = yield* Deferred.make(); + const requestClosed = yield* Deferred.make(); + remote.control.listSeen = () => Deferred.doneUnsafe(requestSeen, Effect.void); + remote.control.listClosed = () => Deferred.doneUnsafe(requestClosed, Effect.void); + const reading = yield* next.runtime + .readThreadSnapshot({ providerThread: next.thread }) + .pipe(Effect.forkScoped); + yield* Deferred.await(requestSeen); + yield* TestClock.adjust("8 seconds"); + yield* Fiber.join(reading); + yield* Deferred.await(requestClosed); + const saved = (yield* restarted.journal.read)[0]!; + assert.equal(saved.admissionRecoveryFailures, attempt); + assert.equal(saved.state, "admission_unknown"); + assert.equal(saved.admissionRecoveryPaused, attempt === 3); + } + assert.isFalse(yield* next.runtime.hasPendingBackgroundWork!); + assert.equal(remote.submissions(), 1); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts index f752ce7b0fc9..5970151aed63 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -281,18 +281,25 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { yield* emit({ type: "node.updated", driver, node: entry.node }); yield* emit({ type: "turn_item.updated", driver, turnItem: entry.item }); }); + yield* Effect.addFinalizer(() => + active ? options.client.forgetAdmission(options.repository, active.messageId) : Effect.void, + ); + const hasBackgroundWork = () => + (!!active && !active.admissionRecoveryPaused) || needsHistoryRestore || monitorSandbox; const finish = Effect.fn("KiloCloudAdapterV2.finish")(function* ( terminal: "completed" | "failed" | "interrupted", resultFailure?: string, ) { if (!active) return; const at = yield* DateTime.now; + const notSubmitted = active.submissionPhase === "preflight"; const saved = { ...active, state: terminal, providerTurn: { ...active.providerTurn, status: terminal, completedAt: at }, }; yield* save(saved); + yield* options.client.forgetAdmission(options.repository, saved.messageId); if (thread?.nativeMetadata?.cloudExecution) { thread = { ...thread, @@ -301,7 +308,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ...thread.nativeMetadata, cloudExecution: { ...thread.nativeMetadata.cloudExecution, - task: saved.remoteState ?? terminal, + task: notSubmitted ? "not_started" : (saved.remoteState ?? terminal), ...(saved.resultStatus ? { result: saved.resultStatus } : {}), }, }, @@ -355,9 +362,15 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { status: "failed", failure: makeProviderFailure({ class: "provider_error", - code: resultFailure ? "kilo_cloud_result_unavailable" : "provider_error", + code: notSubmitted + ? "kilo_cloud_not_submitted" + : resultFailure + ? "kilo_cloud_result_unavailable" + : "provider_error", ...(resultFailure ? { retryable: false } : {}), - message: resultFailure ?? "Kilo Cloud reported a failed task.", + message: notSubmitted + ? "Kilo Cloud preflight failed before submission. No paid request was sent; you can try a new turn." + : (resultFailure ?? "Kilo Cloud reported a failed task."), }), threadDisposition: "reusable", } @@ -382,7 +395,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { // the thread can retry history independently of the ended turn. needsHistoryRestore = false; if (!binding) monitorSandbox = false; - taskState = saved.remoteState ?? terminal; + taskState = notSubmitted ? "not_started" : (saved.remoteState ?? terminal); resultStatus = saved.resultStatus; yield* Deferred.succeed(terminalSignal, undefined); }); @@ -623,6 +636,11 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { yield* emit({ type: "turn_item.updated", driver: driver, turnItem }); }); const reconcile = Effect.fn("KiloCloudAdapterV2.reconcile")(function* () { + if (active?.submissionPhase === "preflight") { + yield* finish("failed"); + return; + } + if (active?.admissionRecoveryPaused) return; const expectedMessageId = active?.messageId; if (!binding) { if (active && !active.prepared) { @@ -630,9 +648,32 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { if (now < admissionProbeAt) return; admissionProbeAt = now + admissionProbeDelay; admissionProbeDelay = Math.min(admissionProbeDelay * 2, 60_000); - const found = yield* wire( - options.client.findAdmission(options.repository, active.messageId), - ); + const found = yield* options.client + .findAdmission(options.repository, active.messageId) + .pipe( + Effect.timeout("8 seconds"), + Effect.catch((cause) => + Effect.gen(function* () { + const failures = (active!.admissionRecoveryFailures ?? 0) + 1; + const paused = + failures >= 3 || + (isCloudError(cause) && + (cause.reason === "recovery_incomplete" || cause.reason === "wrong_owner")); + yield* save({ + ...active!, + admissionRecoveryFailures: failures, + admissionRecoveryPaused: paused, + }); + if (paused) { + monitorSandbox = false; + yield* status( + "Cloud admission recovery is incomplete and automatic scanning is paused. Reopen history to retry reads. Submission and billing remain unknown; do not submit again.", + ); + } + return null; + }), + ), + ); if (found && active?.messageId === expectedMessageId) yield* save({ ...active, prepared: found }); } @@ -646,6 +687,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ), ); yield* save({ ...active, binding, state: "active" }); + yield* options.client.forgetAdmission(options.repository, active!.messageId); } else return; } const intents = yield* wire(options.journal.readThread(thread!.id)); @@ -899,8 +941,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { const ownedBinding = binding; streamFiber = yield* Effect.gen(function* () { // State is changed by the reconciler while this reader observes notifications. - // oxlint-disable-next-line no-unmodified-loop-condition - while (active || needsHistoryRestore || monitorSandbox) { + while (hasBackgroundWork()) { yield* options.client.events(ownedBinding, streamCursor).pipe( Stream.runForEach((event) => Effect.gen(function* () { @@ -924,7 +965,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ), ), ); - if (active || needsHistoryRestore || monitorSandbox) yield* Effect.sleep("5 seconds"); + if (hasBackgroundWork()) yield* Effect.sleep("5 seconds"); } }).pipe( Effect.ensuring( @@ -944,8 +985,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { yield* Effect.gen(function* () { let polls = 0; // Reconcile and lifecycle update this session state. - // oxlint-disable-next-line no-unmodified-loop-condition - while (active || needsHistoryRestore || monitorSandbox) { + while (hasBackgroundWork()) { const pollStartedAt = yield* Clock.currentTimeMillis; yield* watchEvents; if (active || needsHistoryRestore) @@ -962,7 +1002,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ); if (binding && (polls++ % 15 === 0 || !active)) yield* gate.withPermit(lifecycle.pipe(Effect.timeout("10 seconds"), Effect.ignore)); - if (active || needsHistoryRestore || monitorSandbox) + if (hasBackgroundWork()) yield* Effect.raceFirst( Effect.sleep(active ? "2 seconds" : "15 seconds"), Queue.take(wake), @@ -986,8 +1026,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { watching = false; // A turn may arrive while the previous socket is closing. Starting // and retiring the watcher share the turn gate, so its wake is not lost. - if (Exit.isSuccess(exit) && (active || needsHistoryRestore || monitorSandbox)) - yield* watch; + if (Exit.isSuccess(exit) && hasBackgroundWork()) yield* watch; }), ), ), @@ -1046,12 +1085,14 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { return yield* error("Cloud journal belongs to another account or repository."); binding = last?.binding ?? undefined; taskState = - last?.remoteState ?? - (last?.state === "awaiting_result" - ? "completed" - : last?.state === "active" - ? "unknown" - : (last?.state ?? "not_started")); + last?.submissionPhase === "preflight" + ? "not_started" + : (last?.remoteState ?? + (last?.state === "awaiting_result" + ? "completed" + : last?.state === "active" + ? "unknown" + : (last?.state ?? "not_started"))); resultStatus = last?.resultStatus; monitorSandbox = !!last?.binding; active = @@ -1062,6 +1103,11 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ? last : undefined; if (binding) thread = { ...thread, nativeThreadRef: nativeRef(binding.kiloSessionId) }; + if (active?.submissionPhase === "preflight") yield* finish("failed"); + if (active?.admissionRecoveryPaused) + yield* status( + "Cloud admission recovery is paused. Reopen history to retry reads; submission and billing remain unknown.", + ); return thread; }); const unsupported = () => @@ -1072,15 +1118,9 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { providerSessionId: input.providerSessionId, providerSession: session, events: Stream.fromEffectRepeat(Queue.take(queue)), - hasPendingBackgroundWork: Effect.sync( - () => active !== undefined || needsHistoryRestore || monitorSandbox, - ), + hasPendingBackgroundWork: Effect.sync(hasBackgroundWork), hasPendingBackgroundWorkForThread: (candidate) => - Effect.sync( - () => - candidate.id === thread?.id && - (active !== undefined || needsHistoryRestore || monitorSandbox), - ), + Effect.sync(() => candidate.id === thread?.id && hasBackgroundWork()), ensureThread: (request) => gate.withPermit( Effect.gen(function* () { @@ -1113,7 +1153,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { providerTurn: saved.providerTurn, }); binding = saved.binding ?? undefined; - monitorSandbox = true; + monitorSandbox = !!binding; if ( saved.state === "completed" || saved.state === "failed" || @@ -1218,6 +1258,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { binding: binding ?? null, prepared: null, state: "admission_unknown", + submissionPhase: "preflight", interruptRequested: false, answeredRequestIds: [], providerThread, @@ -1267,19 +1308,34 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { driver, providerTurn: intent.providerTurn, }); + // This write is part of the client's dispatch boundary, after its + // credentials/preflight reads and before the paid HTTP execute. + const beforePaidPost = Effect.suspend(() => + save({ ...active!, submissionPhase: "post_attempted" }).pipe( + Effect.asVoid, + Effect.mapError( + () => + new KiloCloudError({ operation: "submission-journal", reason: "rejected" }), + ), + ), + ); let submissionConfirmed = false; yield* Effect.gen(function* () { if (binding) - yield* options.client.send(binding, { - messageId, - prompt: request.message.text, - model, - ...(variant ? { variant } : {}), - }); + yield* options.client.send( + binding, + { + messageId, + prompt: request.message.text, + model, + ...(variant ? { variant } : {}), + }, + beforePaidPost, + ); else { - const prepared = yield* options.client.prepare(payload); + const prepared = yield* options.client.prepare(payload, beforePaidPost); submissionConfirmed = true; - yield* save({ ...intent, prepared }); + yield* save({ ...active!, prepared }); binding = yield* options.client.bind( prepared, options.repository, @@ -1292,7 +1348,8 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { yield* emit({ type: "provider_thread.updated", driver, providerThread: thread }); }).pipe( Effect.catch((cause) => - !submissionConfirmed && isCloudError(cause) && cause.reason === "rejected" + active?.submissionPhase === "preflight" || + (!submissionConfirmed && isCloudError(cause) && cause.reason === "rejected") ? finish("failed") : status( "Cloud admission is uncertain. Its operation ID is saved; no automatic retry will start another paid task.", @@ -1308,6 +1365,10 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { Effect.gen(function* () { yield* owned(request.providerThread); if (!active || active.providerTurn.id !== request.providerTurnId) return false; + if (active.submissionPhase === "preflight") { + yield* finish("interrupted"); + return false; + } if (active.state === "awaiting_result") { yield* save({ ...active, resultStatus: "cancelled" }); yield* finish("interrupted"); @@ -1363,6 +1424,15 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { gate.withPermit( Effect.gen(function* () { yield* owned(request.providerThread); + const resumeAdmissionWatch = active?.admissionRecoveryPaused === true; + if (resumeAdmissionWatch) { + yield* save({ + ...active!, + admissionRecoveryPaused: false, + admissionRecoveryFailures: 0, + }); + admissionProbeAt = 0; + } yield* reconcile().pipe( Effect.timeout("10 seconds"), Effect.mapError(() => @@ -1374,6 +1444,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ), ); yield* lifecycle; + if (resumeAdmissionWatch && hasBackgroundWork()) yield* watch; const intents = yield* wire(options.journal.readThread(thread!.id)); return { providerThread: thread!, diff --git a/apps/server/src/provider/Drivers/KiloDriver.ts b/apps/server/src/provider/Drivers/KiloDriver.ts index 8a97508fcbf0..fc166f7b08ae 100644 --- a/apps/server/src/provider/Drivers/KiloDriver.ts +++ b/apps/server/src/provider/Drivers/KiloDriver.ts @@ -118,6 +118,7 @@ export const KiloDriver: ProviderDriver = { instanceId: continuationKey, binaryPath: input.config.binaryPath, profileDirectory, + processStateDirectory: server.stateDir, environment, authContent, }).pipe( diff --git a/apps/server/src/provider/kilo/KiloCloudClient.ts b/apps/server/src/provider/kilo/KiloCloudClient.ts index ccf0251a1bcc..a67145b1dca0 100644 --- a/apps/server/src/provider/kilo/KiloCloudClient.ts +++ b/apps/server/src/provider/kilo/KiloCloudClient.ts @@ -17,6 +17,7 @@ export class KiloCloudError extends Schema.TaggedError()("KiloCl "invalid_response", "wrong_owner", "unsupported", + "recovery_incomplete", ]), messageId: Schema.optional(Schema.String), }) {} diff --git a/apps/server/src/provider/kilo/KiloCloudJournal.ts b/apps/server/src/provider/kilo/KiloCloudJournal.ts index 1d1635ea5072..9ce1a1a9f473 100644 --- a/apps/server/src/provider/kilo/KiloCloudJournal.ts +++ b/apps/server/src/provider/kilo/KiloCloudJournal.ts @@ -29,6 +29,9 @@ export const CloudIntent = Schema.Struct({ kiloSessionId: Schema.NonEmptyString, }), ), + // Absent on older records: their admission remains uncertain. Only a durable + // preflight marker proves that the paid request has not reached execute. + submissionPhase: Schema.optional(Schema.Literals(["preflight", "post_attempted"])), state: Schema.Literals([ "admission_unknown", "active", @@ -54,6 +57,8 @@ export const CloudIntent = Schema.Struct({ incompleteReplySeen: Schema.optional(Schema.Boolean), }), ), + admissionRecoveryPaused: Schema.optional(Schema.Boolean), + admissionRecoveryFailures: Schema.optional(Schema.Number), interruptRequested: Schema.Boolean, answeredRequestIds: Schema.Array(Schema.String), providerThread: OrchestrationV2ProviderThread, @@ -133,6 +138,7 @@ export const make = Effect.fn("KiloCloudJournal.make")(function* (directory: str prior.messageId !== intent.messageId || prior.payloadHash !== intent.payloadHash || prior.policyHash !== intent.policyHash || + (intent.submissionPhase === "preflight" && prior.submissionPhase !== "preflight") || prior.providerThread.id !== intent.providerThread.id || prior.providerTurn.id !== intent.providerTurn.id || (["completed", "failed", "interrupted"].includes(prior.state) && diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts index f7822d1750dd..8d79dc87fa94 100644 --- a/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts @@ -73,6 +73,64 @@ function json(response: NodeHttp.ServerResponse, data: unknown) { response.end(JSON.stringify({ result: { data } })); } describe("Kilo personal Cloud control-plane customer API", () => { + it("never binds a partial match after bounded candidate failures and drops failed scan state", async () => { + let unavailable = true; + let lists = 0; + let badReads = 0; + const missing = "workspace_00000000-0000-0000-0000-000000000000"; + const { client } = await server((req, res) => { + expect(req.method).toBe("GET"); + const url = new URL(req.url!, "http://localhost"); + const input = JSON.parse(url.searchParams.get("input")!) as Record; + if (url.pathname.endsWith("cliSessionsV2.list")) { + lists++; + return json(res, { + cliSessions: [ + { session_id: "ses_unavailable", cloud_agent_session_id: missing }, + { session_id: session.kiloSessionId, cloud_agent_session_id: session.sessionId }, + ], + nextCursor: null, + }); + } + if (input.cloudAgentSessionId === missing) { + badReads++; + res.writeHead(unavailable ? 503 : 404); + res.end(); + return; + } + return json(res, session); + }); + expect(await run(client.findAdmission(binding.repository, messageId))).toBeNull(); + expect(await run(client.findAdmission(binding.repository, messageId))).toBeNull(); + expect( + (await run(client.findAdmission(binding.repository, messageId).pipe(Effect.flip))).reason, + ).toBe("recovery_incomplete"); + expect(badReads).toBe(3); + expect(lists).toBe(1); + unavailable = false; + expect(await run(client.findAdmission(binding.repository, messageId))).toEqual({ + cloudAgentSessionId: session.sessionId, + kiloSessionId: session.kiloSessionId, + }); + expect(lists).toBe(2); + }); + it("clears abandoned scans and repeated cursors without treating partial reads as absence", async () => { + let repeated = true; + const cursors: Array = []; + const { client } = await server((req, res) => { + const url = new URL(req.url!, "http://localhost"); + const input = JSON.parse(url.searchParams.get("input")!) as Record; + cursors.push(input.cursor); + return json(res, { cliSessions: [], nextCursor: repeated ? "cycle" : null }); + }); + expect( + (await run(client.findAdmission(binding.repository, messageId).pipe(Effect.flip))).reason, + ).toBe("recovery_incomplete"); + repeated = false; + expect(await run(client.findAdmission(binding.repository, messageId))).toBeNull(); + expect(cursors).toEqual([undefined, "cycle", undefined]); + await run(client.forgetAdmission(binding.repository, messageId)); + }); it("continues an uncertain-admission scan across read budgets and cursor pages without resubmitting", async () => { const reads: string[] = []; const cursors: Array = []; @@ -217,7 +275,7 @@ describe("Kilo personal Cloud control-plane customer API", () => { repeated = true; expect( (await run(client.findAdmission(binding.repository, messageId).pipe(Effect.flip))).reason, - ).toBe("invalid_response"); + ).toBe("recovery_incomplete"); }); it("authenticates a customer WebSocket, resumes its cursor and rejects a foreign session event", async () => { const expiresAt = await run(Clock.currentTimeMillis); diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.ts index 2d29d2ee3a05..58f694bd1293 100644 --- a/apps/server/src/provider/kilo/KiloCloudWebClient.ts +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.ts @@ -1,4 +1,5 @@ import * as Effect from "effect/Effect"; +import * as Clock from "effect/Clock"; import * as NodeSocket from "@effect/platform-node/NodeSocket"; import * as Socket from "effect/unstable/socket/Socket"; import * as Redacted from "effect/Redacted"; @@ -184,6 +185,7 @@ export const make = (options: { schema: Schema.Decoder, mutation = false, messageId?: string, + beforePaidPost: Effect.Effect = Effect.void, ) => { const uncertain = () => failure(operation, mutation ? "admission_unknown" : "invalid_response", messageId); @@ -204,9 +206,13 @@ export const make = (options: { }, }, ); - const response = yield* client.execute( - mutation ? HttpClientRequest.bodyText(req, encode(body), "application/json") : req, - ); + const preparedRequest = mutation + ? HttpClientRequest.bodyText(req, encode(body), "application/json") + : req; + // Commit the durable attempt boundary after all local/preflight work. A failed + // commit must prevent execute; after this point an outcome may be uncertain. + yield* beforePaidPost; + const response = yield* client.execute(preparedRequest); if (response.status < 200 || response.status >= 300) return yield* response.status >= 500 || response.status === 408 || response.status === 409 ? uncertain() @@ -313,6 +319,9 @@ export const make = (options: { pending: Array<{ session_id: string; cloud_agent_session_id: string | null }>; deferred: Array<{ session_id: string; cloud_agent_session_id: string | null }>; loaded: boolean; + rounds: number; + failures: number; + touchedAt: number; seenCursors: Set; matches: Map; } @@ -368,9 +377,16 @@ export const make = (options: { findAdmission: (repository: string, initialMessageId: string) => Effect.gen(function* () { const key = `${repository}\0${initialMessageId}`; + const now = yield* Clock.currentTimeMillis; + for (const [cachedKey, cached] of admissionScans) + if (now - cached.touchedAt > 300_000) admissionScans.delete(cachedKey); let scan = admissionScans.get(key); if (!scan) { + if (admissionScans.size >= 64) admissionScans.delete(admissionScans.keys().next().value!); scan = { + rounds: 0, + failures: 0, + touchedAt: now, pending: [], deferred: [], loaded: false, @@ -379,11 +395,14 @@ export const make = (options: { }; admissionScans.set(key, scan); } + scan.touchedAt = now; // At most 25 candidate reads per call. Later polls continue this scan. for (let budget = 25; budget > 0; budget--) { if (!scan.pending.length) { if (scan.loaded && !scan.cursor) { if (scan.deferred.length) { + if (++scan.rounds >= 3) + return yield* failure("reconcile-admission", "recovery_incomplete"); scan.pending = scan.deferred; scan.deferred = []; return null; @@ -412,8 +431,11 @@ export const make = (options: { nextCursor: Schema.NullOr(Schema.String), }), ); - if (page.nextCursor && scan.seenCursors.has(page.nextCursor)) - return yield* failure("reconcile-admission", "invalid_response"); + if ( + page.nextCursor && + (scan.seenCursors.has(page.nextCursor) || scan.seenCursors.size >= 100) + ) + return yield* failure("reconcile-admission", "recovery_incomplete"); if (page.nextCursor) scan.seenCursors.add(page.nextCursor); scan.cursor = page.nextCursor ?? undefined; scan.loaded = true; @@ -454,17 +476,38 @@ export const make = (options: { }); } return null; + }).pipe( + Effect.catchTag("KiloCloudError", (cause) => + Effect.gen(function* () { + const key = `${repository}\0${initialMessageId}`; + const scan = admissionScans.get(key); + if (scan && ++scan.failures >= 3) { + admissionScans.delete(key); + return yield* failure("reconcile-admission", "recovery_incomplete"); + } + if (cause.reason === "recovery_incomplete" || cause.reason === "wrong_owner") + admissionScans.delete(key); + return yield* cause; + }), + ), + ), + forgetAdmission: (repository: string, initialMessageId: string) => + Effect.sync(() => { + admissionScans.delete(`${repository}\0${initialMessageId}`); }), /** Admission is paid. Persist operationKey and initialMessageId before calling; no retries here. */ - prepare: (input: { - readonly operationKey: string; - readonly initialMessageId: string; - readonly prompt: string; - readonly repository: string; - readonly branch: string; - readonly model: string; - readonly variant?: string; - }) => + prepare: ( + input: { + readonly operationKey: string; + readonly initialMessageId: string; + readonly prompt: string; + readonly repository: string; + readonly branch: string; + readonly model: string; + readonly variant?: string; + }, + beforePaidPost: Effect.Effect = Effect.void, + ) => preflight(input.repository).pipe( Effect.andThen( request( @@ -491,6 +534,7 @@ export const make = (options: { Prepared, true, input.initialMessageId, + beforePaidPost, ), ), ), @@ -526,6 +570,7 @@ export const make = (options: { readonly model: string; readonly variant?: string; }, + beforePaidPost: Effect.Effect = Effect.void, ) => check(binding).pipe( Effect.andThen( @@ -547,6 +592,7 @@ export const make = (options: { Sent, true, input.messageId, + beforePaidPost, ), ), Effect.flatMap((sent) => diff --git a/apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs b/apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs index f24ba880dfed..2d302448401f 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs +++ b/apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs @@ -13,6 +13,7 @@ await Effect.runPromise( instanceId: "crash-fixture", binaryPath: process.argv[2], profileDirectory: process.argv[3], + ...(process.argv[4] ? { processStateDirectory: process.argv[4] } : {}), environment: { PATH: process.env.PATH, HOME: process.argv[3], diff --git a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts index c1272162e9ca..bc416ccaf27b 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts @@ -15,6 +15,7 @@ import * as Scope from "effect/Scope"; import * as Schema from "effect/Schema"; import { describe } from "vite-plus/test"; +import * as ServerLedger from "../OpenCodeServerLedger.ts"; import * as KiloRuntime from "./KiloRuntime.ts"; import { KiloDriver } from "../Drivers/KiloDriver.ts"; import * as ServerConfig from "../../config.ts"; @@ -254,96 +255,113 @@ describe.skipIf(!binary)("KiloRuntime native lifecycle", () => { { timeout: 30000 }, ); - it.live.skipIf(platform !== "linux")( - "reaps a real Kilo process after its T3 owner is killed and resumes its session", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const profile = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-crash-" }); - let group: number | undefined; - const owner = yield* Effect.acquireRelease( - Effect.sync(() => - NodeChildProcess.spawn( - process.execPath, - [ - NodeURL.fileURLToPath(new URL("./KiloRuntime.crash.fixture.mjs", import.meta.url)), - binary!, - profile, - ], - { stdio: ["ignore", "ignore", "ignore", "ipc"] }, + for (const globalLedger of [false, true]) + it.live.skipIf(platform !== "linux")( + `reaps a real Kilo owner crash, including moved profiles: globalLedger=${globalLedger}`, + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-crash-" }); + const profile = path.join(root, "profile"); + yield* fs.makeDirectory(profile); + const processStateDirectory = path.join(root, "server-state"); + const ledgerDir = path.join( + globalLedger ? processStateDirectory : path.join(profile, "t3-processes"), + "opencode-servers", + ); + let group: number | undefined; + const owner = yield* Effect.acquireRelease( + Effect.sync(() => + NodeChildProcess.spawn( + process.execPath, + [ + NodeURL.fileURLToPath( + new URL("./KiloRuntime.crash.fixture.mjs", import.meta.url), + ), + binary!, + profile, + ...(globalLedger ? [processStateDirectory] : []), + ], + { stdio: ["ignore", "ignore", "ignore", "ipc"] }, + ), ), - ), - (child) => - Effect.sync(() => { - child.kill("SIGKILL"); - if (group !== undefined) { - try { - process.kill(-group, "SIGKILL"); - } catch { - /* already stopped */ + (child) => + Effect.sync(() => { + child.kill("SIGKILL"); + if (group !== undefined) { + try { + process.kill(-group, "SIGKILL"); + } catch { + /* already stopped */ + } } - } - }), - ); - const message = yield* Effect.promise( - () => - new Promise<{ - pid: number; - session: { instanceId: string; sessionId: string; directory: string }; - }>((resolve, reject) => { - owner.on("message", (value) => { - const message = value as { - type: string; - pid: number; - session: { instanceId: string; sessionId: string; directory: string }; - }; - group = message.pid; - if (message.type === "ready") resolve(message); - }); - owner.once("exit", () => - reject(new Error("Kilo crash fixture exited before readiness")), - ); - owner.once("error", reject); - }), - ); - const entries = yield* fs.readDirectory( - path.join(profile, "t3-processes", "opencode-servers"), - ); - assert.equal(entries.length, 1); - const recorded = yield* decodeOwner( - yield* fs.readFileString( - path.join(profile, "t3-processes", "opencode-servers", entries[0]!), - ), - ); - assert.equal(recorded.owner.pid, owner.pid); - assert.equal(recorded.pgid, group); - const exited = NodeEvents.EventEmitter.once(owner, "exit"); - owner.kill("SIGKILL"); - yield* Effect.promise(() => exited); - const running = (pid: number) => - fs.readFileString(`/proc/${pid}/stat`).pipe( - Effect.map((stat) => !stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")), - Effect.orElseSucceed(() => false), + }), ); - assert.isTrue(yield* running(message.pid)); - const restarted = yield* KiloRuntime.make({ - instanceId: "crash-fixture", - binaryPath: binary!, - profileDirectory: profile, - environment: { ...environment, HOME: profile }, - }); - assert.isFalse(yield* running(message.pid)); - assert.deepEqual( - yield* fs.readDirectory(path.join(profile, "t3-processes", "opencode-servers")), - [], - ); - const fresh = yield* restarted.open(profile); - assert.equal((yield* fresh.client.read(message.session)).id, message.session.sessionId); - assert.isTrue(yield* fresh.isRunning); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - { timeout: 30000 }, - ); + const message = yield* Effect.promise( + () => + new Promise<{ + pid: number; + session: { instanceId: string; sessionId: string; directory: string }; + }>((resolve, reject) => { + owner.on("message", (value) => { + const message = value as { + type: string; + pid: number; + session: { instanceId: string; sessionId: string; directory: string }; + }; + group = message.pid; + if (message.type === "ready") resolve(message); + }); + owner.once("exit", () => + reject(new Error("Kilo crash fixture exited before readiness")), + ); + owner.once("error", reject); + }), + ); + const entries = yield* fs.readDirectory(ledgerDir); + assert.equal(entries.length, 1); + const recorded = yield* decodeOwner( + yield* fs.readFileString(path.join(ledgerDir, entries[0]!)), + ); + assert.equal(recorded.owner.pid, owner.pid); + assert.equal(recorded.pgid, group); + const exited = NodeEvents.EventEmitter.once(owner, "exit"); + owner.kill("SIGKILL"); + yield* Effect.promise(() => exited); + const running = (pid: number) => + fs.readFileString(`/proc/${pid}/stat`).pipe( + Effect.map((stat) => !stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")), + Effect.orElseSucceed(() => false), + ); + assert.isTrue(yield* running(message.pid)); + const replacementProfile = globalLedger ? path.join(root, "moved-profile") : profile; + if (globalLedger) { + yield* fs.rename(profile, replacementProfile); + // This is the same server-global startup reaper. No account/profile + // lookup is needed, so removal from settings cannot hide the process. + const ledger = yield* ServerLedger.make({ stateDir: processStateDirectory }); + yield* ledger.reapOrphans; + yield* fs.makeDirectory(profile); + } + const restarted = yield* KiloRuntime.make({ + instanceId: "crash-fixture", + binaryPath: binary!, + profileDirectory: replacementProfile, + environment: { ...environment, HOME: profile }, + }); + assert.isFalse(yield* running(message.pid)); + assert.deepEqual(yield* fs.readDirectory(ledgerDir), []); + const fresh = yield* restarted.open(profile); + if (globalLedger) { + const newSession = yield* fresh.client.create([]); + assert.notEqual(newSession.sessionId, message.session.sessionId); + } else + assert.equal((yield* fresh.client.read(message.session)).id, message.session.sessionId); + assert.isTrue(yield* fresh.isRunning); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + { timeout: 30000 }, + ); it.live( "cleans failed startup and can open a fresh process afterward", diff --git a/apps/server/src/provider/kilo/KiloRuntime.ts b/apps/server/src/provider/kilo/KiloRuntime.ts index 9a5429023824..137895ddf06e 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.ts @@ -92,6 +92,8 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { readonly profileDirectory: string; readonly environment: NodeJS.ProcessEnv; readonly authContent?: string; + /** Stable T3 state directory; process ownership must survive profile removal. */ + readonly processStateDirectory?: string; }) { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fs = yield* FileSystem.FileSystem; @@ -102,8 +104,16 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { const fail = (operation: string, detail: string) => (cause: unknown) => new KiloRuntimeError({ operation, detail, cause }); const profile = path.resolve(input.profileDirectory); - const ledger = yield* ServerLedger.make({ stateDir: path.join(profile, "t3-processes") }); - yield* ledger.reapOrphans; + const ledger = yield* ServerLedger.make({ + stateDir: input.processStateDirectory ?? path.join(profile, "t3-processes"), + }); + if (input.processStateDirectory) { + // The server-global ledger also reaps at startup, even when no Kilo profile + // remains configured. Do not block model discovery on orphan shutdown. + yield* ledger.reapOrphans.pipe(Effect.forkIn(owner)); + const legacy = yield* ServerLedger.make({ stateDir: path.join(profile, "t3-processes") }); + yield* legacy.reapOrphans.pipe(Effect.forkIn(owner)); + } else yield* ledger.reapOrphans; const authContent = input.authContent ?? (yield* readAuth(profile, input.environment)); const environment: NodeJS.ProcessEnv = { ...input.environment, diff --git a/apps/web/src/components/ChatMarkdown.tsx b/apps/web/src/components/ChatMarkdown.tsx index 4b6e3740037d..07173865d960 100644 --- a/apps/web/src/components/ChatMarkdown.tsx +++ b/apps/web/src/components/ChatMarkdown.tsx @@ -2925,6 +2925,7 @@ const CHAT_MARKDOWN_COMPONENTS = { ); }, a: function MarkdownAnchor({ node, href, children, title: _title, ...props }) { + const localWorkspaceEnabled = use(ChatMarkdownLocalWorkspaceContext); const { cwd, environmentId, @@ -3027,7 +3028,8 @@ const CHAT_MARKDOWN_COMPONENTS = { // the panel it opens offers the browser as one of its actions. if ( !href || - openChangeRequestLink(event, href, undefined, environmentId ?? undefined) + (localWorkspaceEnabled && + openChangeRequestLink(event, href, undefined, environmentId ?? undefined)) ) { return; } @@ -3130,6 +3132,7 @@ const CHAT_MARKDOWN_COMPONENTS = { target={pullRequestPreviewTarget} confirmBeforeOpen={confirmBeforeOpen} onOpenPullRequest={(targetUrl) => + localWorkspaceEnabled && openChangeRequestLink( { metaKey: false, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 727b7feb1944..e9c4ce9329ed 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -1,3 +1,4 @@ +import { cloudExecutionLabel } from "@t3tools/client-runtime/cloudExecutionLabels"; import { ChatCanvas } from "./chat/ChatCanvas"; import { usageLimitRecoveryBannerItem } from "./chat/UsageLimitRecoveryBanner"; import { @@ -10723,9 +10724,12 @@ export default function ChatView(props: ChatViewProps) { {cloudExecution ? (

Last observation: {cloudExecution.observedAt ?? "unavailable"}. Task:{" "} - {cloudExecution.task}.{" "} - {cloudExecution.result ? `Result: ${cloudExecution.result}. ` : ""}Sandbox:{" "} - {cloudExecution.sandbox}. Compute: {cloudExecution.billing} + {cloudExecutionLabel(cloudExecution.task)}.{" "} + {cloudExecution.result + ? `Result: ${cloudExecutionLabel(cloudExecution.result)}. ` + : ""} + Sandbox: {cloudExecutionLabel(cloudExecution.sandbox)}. Compute:{" "} + {cloudExecutionLabel(cloudExecution.billing)} {cloudExecution.billingAttribution === "payer_shared" ? " (shared account)" : ""} diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md index 834dd6e65dfb..ed4790f6ced0 100644 --- a/docs/user/providers-kilo.md +++ b/docs/user/providers-kilo.md @@ -10,6 +10,7 @@ thread. Saved history remains available. Like T3's OpenCode provider, Kilo trusts native runtime configuration, plugins and MCP servers. Only open repositories and profiles whose configuration you trust. +Trusted native configuration and plugins can access this profile's credentials. Configured MCP processes or connections can start before a model tool call or approval. Supervised and Plan modes govern supported tool calls; they are not an OS sandbox and do not isolate native configuration or MCP initialization. @@ -19,24 +20,28 @@ Kilo 7.8.3 loads legacy `.kilo/mcp.json` and `.kilocode/mcp.json` even when not all MCP loading. T3 does not force either flag as a security boundary, rewrite configuration, or patch the installed runtime. Explicit native settings remain trusted. Background subagents stay disabled. Foreground child agents require Full -access because Kilo does not inherit parent `ask` rules reliably. +access with the default interaction mode. Plan mode disables them, even with Full +access, because Kilo does not inherit parent `ask` rules reliably. T3 stops owned processes on normal shutdown. On Linux and macOS, it records their -process identity and reaps processes from a dead T3 owner when the same profile -is reopened. Cleanup checks the recorded PID, start time, command and owner; +process identity in T3's state directory and reaps processes from a dead T3 owner +on server startup, including when that account profile was removed or moved. Cleanup checks the recorded PID, start time, command and owner; it never kills by executable name. Crash recovery on macOS and native Windows -process cleanup have not been verified in this environment. - -| Capability | Local Kilo | Kilo Cloud | -| ------------------------------------------ | --------------------------------------------------------- | --------------------------------------------------------------------------------------- | -| Prompts and follow-up | Native streaming, tools and reasoning | Full access; history updates, no token-streaming claim | -| Concurrent sessions | Separate processes and account profiles | Separate task identities and remote worktrees; runs alongside local sessions | -| History and recovery | Native history and resume | Durable admission and result recovery; no blind paid resubmission | -| Stop | Native abort and owned-process cleanup | Inference interrupt while running; local retrieval cancellation after remote completion | -| Approvals and questions | Native supported tool approvals and questions | Handles emitted interactions; cannot enforce restricted policy | -| Rewind, fork, checkpoints, text generation | Integrated with native sessions and T3 checkpoints | Not supported | -| Subagents | Foreground Full access only; restricted modes denied | Remote Full access may run them; child history not integrated | -| Clients | Web/desktop/mobile selection, models, status and controls | Account/repository/model settings and task status; local workspace actions disabled | +process cleanup have not been verified in this environment. Older profile-local +records are recovered only if that original profile is reopened. Deleting or moving +T3's own state directory can lose cleanup records; T3 never guesses ownership from +a process name. + +| Capability | Local Kilo | Kilo Cloud | +| ------------------------------------------ | -------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| Prompts and follow-up | Native streaming, tools and reasoning | Full access; history updates, no token-streaming claim | +| Concurrent sessions | Separate processes and account profiles | Separate task identities and remote worktrees; runs alongside local sessions | +| History and recovery | Native history and resume | Durable admission and result recovery; no blind paid resubmission | +| Stop | Native abort and owned-process cleanup | Inference interrupt while running; local retrieval cancellation after remote completion | +| Approvals and questions | Native supported tool approvals and questions | Handles emitted interactions; cannot enforce restricted policy | +| Rewind, fork, checkpoints, text generation | Integrated with native sessions and T3 checkpoints | Not supported | +| Subagents | Foreground Full access and default mode only; restricted/Plan denied | Remote Full access may run them; child history not integrated | +| Clients | Web/desktop/mobile selection, models, status and controls | Account/repository/model settings and task status; local workspace actions disabled | ## Cloud execution and costs @@ -87,3 +92,10 @@ requests remote interruption and waits for confirmation; billing remains separat Local/cloud concurrency and recovery are covered by actual local CLI sessions and loopback customer-contract tests. These do not replace live verification of every deployed cloud behavior or native platform testing. + +If preflight fails before the paid request is attempted, T3 ends that turn locally +and permits an explicit new turn. Interrupted or older admission records without +proof of that boundary remain uncertain. A timeout or an incomplete search never +permits automatic resubmission. Repeatedly unreadable admission candidates pause +automatic scanning; reopening history retries only reads. Remote task and billing +status remain unknown until Kilo confirms the original operation. diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index 40add7a6bc51..15caa42b7d0e 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -366,6 +366,10 @@ "./kilo-icon": { "types": "./src/kiloIcon.ts", "default": "./src/kiloIcon.ts" + }, + "./cloudExecutionLabels": { + "types": "./src/cloudExecutionLabels.ts", + "default": "./src/cloudExecutionLabels.ts" } }, "scripts": { diff --git a/packages/client-runtime/src/cloudExecutionLabels.ts b/packages/client-runtime/src/cloudExecutionLabels.ts new file mode 100644 index 000000000000..81b1d3edf998 --- /dev/null +++ b/packages/client-runtime/src/cloudExecutionLabels.ts @@ -0,0 +1,37 @@ +import type { OrchestrationV2ProviderThread } from "@t3tools/contracts"; + +type Execution = NonNullable< + NonNullable["cloudExecution"] +>; +const labels: Record< + | Execution["task"] + | Execution["sandbox"] + | Execution["billing"] + | NonNullable, + string +> = { + not_started: "Not submitted", + admission_unknown: "Submission unconfirmed", + queued: "Queued", + running: "Running", + completed: "Completed", + failed: "Failed", + interrupted: "Interrupted", + unknown: "Unknown", + active: "Active", + sleeping: "Sleeping", + starting: "Starting", + stopping: "Stopping", + error: "Error", + unreachable: "Unreachable", + idle: "Idle", + settling: "Charges settling", + awaiting_result: "Retrieving output", + available: "Available", + unavailable: "Unavailable", + cancelled: "Retrieval cancelled", +}; + +export function cloudExecutionLabel(value: keyof typeof labels): string { + return labels[value]; +} From 179daefddda140e28acf2ccf40bd6e24c60cb93d Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sun, 4 Oct 2026 10:31:50 +0000 Subject: [PATCH 18/44] fix(kilo): preserve recovery bounds and local workspaces --- .../src/components/LocalWorkspaceNotice.tsx | 43 +-- .../features/files/ThreadFilesRouteScreen.tsx | 14 +- .../src/features/review/ReviewSheet.tsx | 5 +- .../terminal/ThreadTerminalRouteScreen.tsx | 3 +- .../src/state/threadLocalWorkspace.test.ts | 6 + apps/mobile/src/state/threadLocalWorkspace.ts | 30 +- .../src/state/use-selected-thread-worktree.ts | 5 +- .../Adapters/KiloCloudAdapterV2.test.ts | 258 ++++++++++++++++++ .../Adapters/KiloCloudAdapterV2.ts | 243 +++++++++++------ .../src/provider/kilo/KiloCloudClient.ts | 2 + .../src/provider/kilo/KiloCloudJournal.ts | 1 + .../provider/kilo/KiloCloudWebClient.test.ts | 104 ++++++- .../src/provider/kilo/KiloCloudWebClient.ts | 27 +- .../provider/kilo/KiloRuntime.live.test.ts | 6 +- apps/server/src/provider/kilo/KiloRuntime.ts | 11 +- apps/web/src/components/ChatView.tsx | 8 +- docs/user/providers-kilo.md | 8 +- 17 files changed, 630 insertions(+), 144 deletions(-) diff --git a/apps/mobile/src/components/LocalWorkspaceNotice.tsx b/apps/mobile/src/components/LocalWorkspaceNotice.tsx index d167ffbd34f5..dd67144995b2 100644 --- a/apps/mobile/src/components/LocalWorkspaceNotice.tsx +++ b/apps/mobile/src/components/LocalWorkspaceNotice.tsx @@ -1,3 +1,4 @@ +import { NativeStackScreenOptions } from "../native/StackHeader"; import { View } from "react-native"; import { EmptyState } from "./EmptyState"; import { LoadingScreen } from "./LoadingScreen"; @@ -5,28 +6,34 @@ import type { threadLocalWorkspace } from "../state/threadLocalWorkspace"; export function LocalWorkspaceNotice({ state, + title, }: { + readonly title: string; readonly state: ReturnType["localWorkspaceState"]; }) { - if (state === "loading") return ; return ( - - + + + {state === "loading" ? ( + + ) : ( + + )} ); } diff --git a/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx b/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx index 610e87f648c9..37e622ff2c81 100644 --- a/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx +++ b/apps/mobile/src/features/files/ThreadFilesRouteScreen.tsx @@ -492,8 +492,8 @@ export function ThreadFilesTreeScreen(props: ThreadFilesRouteScreenProps) { } }, [cwd, fileInspector.supported, showAuxiliaryPane]); - if (threadId !== null && ["cloud", "error", "unavailable"].includes(localWorkspaceState)) - return ; + if (threadId !== null && localWorkspaceState !== "local" && localWorkspaceState !== "loading") + return ; if (selectedThread === null || environmentId === null || threadId === null) { if (fileInspector.supported) { @@ -509,7 +509,7 @@ export function ThreadFilesTreeScreen(props: ThreadFilesRouteScreenProps) { } if (cwd === null) { - return ; + return ; } if (fileInspector.supported) { @@ -852,18 +852,18 @@ export function ThreadFileScreen(props: ThreadFileRouteScreenProps) { handleReturnToThread(); }, [handleReturnToThread, navigation]); + if (threadId !== null && localWorkspaceState !== "local" && localWorkspaceState !== "loading") + return ; + // A file opened from a project draft has no thread, and needs none: the thread only supplies // the workspace to read from and the target to navigate back to, both of which a draft names // for itself. Wait only for what this file actually cannot render without. - if (threadId !== null && ["cloud", "error", "unavailable"].includes(localWorkspaceState)) - return ; - if (environmentId === null || (threadId !== null && selectedThread === null)) { return ; } if (cwd === null) { - return ; + return ; } if (relativePath === null) { diff --git a/apps/mobile/src/features/review/ReviewSheet.tsx b/apps/mobile/src/features/review/ReviewSheet.tsx index 9050561475b7..d3f4ffb9085d 100644 --- a/apps/mobile/src/features/review/ReviewSheet.tsx +++ b/apps/mobile/src/features/review/ReviewSheet.tsx @@ -448,14 +448,15 @@ type ReviewSheetProps = StaticScreenProps<{ }>; export function ReviewSheet(props: ReviewSheetProps) { + useAdaptiveWorkspacePaneRole("inspector"); const { localWorkspaceEnabled, localWorkspaceState } = useSelectedThreadWorktree(); - if (!localWorkspaceEnabled) return ; + if (!localWorkspaceEnabled) + return ; return ; } function LocalReviewSheet(props: ReviewSheetProps) { const { nativeReviewDiffStyle } = useAppearanceCodeSurface(); - useAdaptiveWorkspacePaneRole("inspector"); const { panes, showAuxiliaryPane } = useAdaptiveWorkspaceLayout(); const navigation = useNavigation(); const insets = useSafeAreaInsets(); diff --git a/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx b/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx index 37cd74bfd9d9..94c9533d5432 100644 --- a/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx +++ b/apps/mobile/src/features/terminal/ThreadTerminalRouteScreen.tsx @@ -243,7 +243,8 @@ type ThreadTerminalRouteScreenProps = StaticScreenProps<{ export function ThreadTerminalRouteScreen(props: ThreadTerminalRouteScreenProps) { const { localWorkspaceEnabled, localWorkspaceState } = useSelectedThreadWorktree(); - if (!localWorkspaceEnabled) return ; + if (!localWorkspaceEnabled) + return ; return ; } diff --git a/apps/mobile/src/state/threadLocalWorkspace.test.ts b/apps/mobile/src/state/threadLocalWorkspace.test.ts index 8e72f532c1fa..27644e7ce23b 100644 --- a/apps/mobile/src/state/threadLocalWorkspace.test.ts +++ b/apps/mobile/src/state/threadLocalWorkspace.test.ts @@ -44,6 +44,12 @@ function expectBlocked(input: Parameters[0], state: }); } describe("mobile thread workspace routing", () => { + it("keeps a resolved local workspace mounted across reconnect errors", () => { + expect(threadLocalWorkspace({ ...local, loadError: "Reconnecting" })).toEqual( + threadLocalWorkspace(local), + ); + expectBlocked({ ...local, detailLoaded: false, loadError: "Reconnect failed" }, "error"); + }); it("restores actual local Kilo worktree, repository and draft flows after hydration", () => { expectBlocked({ ...local, providerThreads: [], detailLoaded: false }, "loading"); expect(threadLocalWorkspace(local)).toEqual({ diff --git a/apps/mobile/src/state/threadLocalWorkspace.ts b/apps/mobile/src/state/threadLocalWorkspace.ts index 60d50245cf56..6596dd19e901 100644 --- a/apps/mobile/src/state/threadLocalWorkspace.ts +++ b/apps/mobile/src/state/threadLocalWorkspace.ts @@ -18,21 +18,29 @@ export function threadLocalWorkspace(input: { active?.driver === "kilo-cloud" || !!active?.nativeMetadata?.cloudExecution; const resolved = input.activeProviderThreadId === null || active !== undefined; + const local = + resolved && + input.driver !== undefined && + input.detailLoaded !== false && + input.providerConfigLoaded !== false && + !!(input.worktreePath ?? input.workspaceRoot); const state = cloud ? "cloud" : input.threadDeleted ? "unavailable" - : input.loadError - ? "error" - : input.detailLoaded === false || input.providerConfigLoaded === false - ? "loading" - : !resolved || input.driver === undefined - ? input.providerConfigLoaded === true && input.detailLoaded === true - ? "unavailable" - : "loading" - : !(input.worktreePath ?? input.workspaceRoot) - ? "unavailable" - : "local"; + : local + ? "local" + : input.loadError + ? "error" + : input.detailLoaded === false || input.providerConfigLoaded === false + ? "loading" + : !resolved || input.driver === undefined + ? input.providerConfigLoaded === true && input.detailLoaded === true + ? "unavailable" + : "loading" + : !(input.worktreePath ?? input.workspaceRoot) + ? "unavailable" + : "local"; const enabled = state === "local"; return { localWorkspaceState: state, diff --git a/apps/mobile/src/state/use-selected-thread-worktree.ts b/apps/mobile/src/state/use-selected-thread-worktree.ts index 811367dde508..b27448ef9023 100644 --- a/apps/mobile/src/state/use-selected-thread-worktree.ts +++ b/apps/mobile/src/state/use-selected-thread-worktree.ts @@ -33,13 +33,14 @@ export function useSelectedThreadWorktree() { [detailWorktreePath, selectedThread?.worktreePath], ); + const serverConfig = selectedEnvironmentRuntime?.serverConfig ?? config.data?.config; return threadLocalWorkspace({ - driver: config.data?.config.providers.find( + driver: serverConfig?.providers.find( (provider) => provider.instanceId === selectedThread?.providerInstanceId, )?.driver, detailLoaded: projection !== null, threadDeleted: detail.status === "deleted", - providerConfigLoaded: config.data !== null, + providerConfigLoaded: serverConfig != null, loadError: Option.getOrNull(detail.error) ?? config.error ?? diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index a702928b36b4..23adab450775 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -4,6 +4,7 @@ import * as KiloRuntime from "../../provider/kilo/KiloRuntime.ts"; import * as KiloAdapter from "./KiloAdapterV2.ts"; import * as NodeHttp from "node:http"; import * as NodeFS from "node:fs"; +import * as NodeSqlite from "node:sqlite"; import * as Account from "../../provider/kilo/KiloCloudAccount.ts"; import * as DateTime from "effect/DateTime"; import * as Clock from "effect/Clock"; @@ -84,6 +85,9 @@ const fixture = Effect.acquireRelease( signalInterrupt = resolve; }); const control = { + prepareStatus: 200, + preparePosts: 0, + listReads: 0, listStatus: 200, parkList: false, listSeen: undefined as (() => void) | undefined, @@ -191,6 +195,12 @@ const fixture = Effect.acquireRelease( return reply(control.malformedPreflight ? {} : []); } if (operation === "cloudAgentNext.prepareSession") { + control.preparePosts++; + if (control.prepareStatus !== 200) { + response.writeHead(control.prepareStatus); + response.end(); + return; + } submissions++; completeLocal(); const suffix = String(submissions).padStart(12, "0"); @@ -209,6 +219,7 @@ const fixture = Effect.acquireRelease( } return reply({ cloudAgentSessionId: state.cloud, kiloSessionId: state.native }); } + if (operation === "cliSessionsV2.list") control.listReads++; if (operation === "cliSessionsV2.list" && control.parkList) { response.once("close", () => control.listClosed?.()); response.on("error", () => {}); @@ -1680,6 +1691,7 @@ it.live( assert.isFalse(yield* recovered.runtime.hasPendingBackgroundWork!); remote.control.listStatus = 200; remote.control.resultReads = 0; + remote.control.status = "running"; // Inference can complete while sandbox and billing remain active. remote.control.completeAfterResultReads = 2; const done = yield* Deferred.make(); yield* recovered.runtime.events.pipe( @@ -1694,6 +1706,14 @@ it.live( assert.equal(complete.state, "completed"); assert.isFalse(complete.admissionRecoveryPaused); assert.isAtLeast(remote.control.resultReads, 2); + assert.isTrue(yield* recovered.runtime.hasPendingBackgroundWork!); + remote.control.status = "completed"; + const sleeping = yield* recovered.runtime.readThreadSnapshot({ + providerThread: recovered.thread, + }); + assert.equal(sleeping.providerThread.nativeMetadata?.cloudExecution?.sandbox, "sleeping"); + assert.equal(sleeping.providerThread.nativeMetadata?.cloudExecution?.billing, "idle"); + assert.isFalse(yield* recovered.runtime.hasPendingBackgroundWork!); assert.equal(remote.submissions(), 1); assert.equal(remote.control.sendPosts, 0); }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), @@ -1744,3 +1764,241 @@ it.effect( }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), 20_000, ); + +const uncertainAdmission = Effect.gen(function* () { + const remote = yield* fixture; + remote.control.dropNextPrepare = true; + remote.control.hideAdmissions = true; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const harness = yield* admissionHarness(remote, directory); + const firstScope = yield* Scope.fork(yield* Effect.scope); + const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, firstScope)); + yield* first.runtime.startTurn(harness.turn(first.thread)); + yield* Scope.close(firstScope, Exit.void); + return { remote, directory }; +}); + +it.live( + "bounds recovery when real SQLite UPDATEs fail and retains the paid reservation", + () => + Effect.gen(function* () { + const { remote, directory } = yield* uncertainAdmission; + remote.control.listStatus = 503; + const harness = yield* admissionHarness(remote, directory); + const opened = yield* harness.open; + const db = yield* Effect.acquireRelease( + Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), + (db) => Effect.sync(() => db.close()), + ); + db.exec( + "CREATE TRIGGER fail_recovery_save BEFORE UPDATE ON intents BEGIN SELECT RAISE(FAIL, 'fixture write failure'); END", + ); + const baseClock = yield* Clock.Clock; + const now = yield* Clock.currentTimeMillis; + for (let attempt = 0; attempt < 3; attempt++) { + yield* opened.runtime + .readThreadSnapshot({ providerThread: opened.thread }) + .pipe( + Effect.provideService(Clock.Clock, clockAt(baseClock, now + attempt * 61_000)), + Effect.ignore, + ); + } + assert.isFalse(yield* opened.runtime.hasPendingBackgroundWork!); + const saved = (yield* harness.journal.read)[0]!; + assert.equal(saved.state, "admission_unknown"); + assert.equal(saved.submissionPhase, "post_attempted"); + yield* opened.runtime + .interruptTurn({ providerThread: opened.thread, providerTurnId: saved.providerTurn.id }) + .pipe(Effect.flip); + yield* opened.runtime.startTurn(harness.turn(opened.thread, 2)).pipe(Effect.flip); + assert.equal(remote.control.preparePosts, 1); + // Persisted uncertainty survives a new adapter even though the failed disk + // could not persist the local pause. No paid retry is permitted. + const restarted = yield* admissionHarness(remote, directory); + const next = yield* restarted.open; + yield* next.runtime.startTurn(restarted.turn(next.thread, 2)).pipe(Effect.flip); + db.exec("DROP TRIGGER fail_recovery_save"); + remote.control.listStatus = 200; + remote.control.hideAdmissions = false; + const recovered = yield* opened.runtime.readThreadSnapshot({ providerThread: opened.thread }); + assert.equal(recovered.providerThread.nativeMetadata?.cloudExecution?.task, "completed"); + assert.equal((yield* harness.journal.read)[0]?.state, "completed"); + assert.equal(remote.control.preparePosts, 1); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live( + "adopts another adapter's durable recovery before probing with a stale revision", + () => + Effect.gen(function* () { + const { remote, directory } = yield* uncertainAdmission; + remote.control.hideAdmissions = false; + remote.control.status = "running"; + const a = yield* admissionHarness(remote, directory); + const b = yield* admissionHarness(remote, directory); + const first = yield* a.open; + const winner = yield* b.open; + yield* winner.runtime.readThreadSnapshot({ providerThread: winner.thread }); + const reads = remote.control.listReads; + const adopted = yield* first.runtime.readThreadSnapshot({ providerThread: first.thread }); + assert.equal(remote.control.listReads, reads); + assert.equal(adopted.providerThread.nativeMetadata?.cloudExecution?.task, "running"); + assert.equal((yield* a.journal.read)[0]?.state, "active"); + remote.control.interruptAccepted = true; + const saved = (yield* a.journal.read)[0]!; + yield* first.runtime.interruptTurn({ + providerThread: first.thread, + providerTurnId: saved.providerTurn.id, + }); + yield* winner.runtime.readThreadSnapshot({ providerThread: winner.thread }); + assert.equal((yield* b.journal.read)[0]?.state, "interrupted"); + assert.equal(remote.control.interruptPosts, 1); + assert.equal(remote.control.preparePosts, 1); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live( + "resets admission failures on progress and resets the retry cadence after manual resume", + () => + Effect.gen(function* () { + const { remote, directory } = yield* uncertainAdmission; + const harness = yield* admissionHarness(remote, directory); + const opened = yield* harness.open; + const baseClock = yield* Clock.Clock; + const now = yield* Clock.currentTimeMillis; + const readAt = (ms: number) => + opened.runtime + .readThreadSnapshot({ providerThread: opened.thread }) + .pipe(Effect.provideService(Clock.Clock, clockAt(baseClock, now + ms))); + for (const [index, status] of [503, 200, 503].entries()) { + remote.control.listStatus = status; + yield* readAt(index * 61_000); + assert.equal( + (yield* harness.journal.read)[0]?.admissionRecoveryFailures, + status === 200 ? 0 : 1, + ); + assert.isFalse((yield* harness.journal.read)[0]?.admissionRecoveryPaused); + } + yield* readAt(183_000); + yield* readAt(244_000); + assert.isTrue((yield* harness.journal.read)[0]?.admissionRecoveryPaused); + remote.control.listStatus = 200; + yield* readAt(305_000); // Explicit resume; a successful empty scan is still uncertain. + const reads = remote.control.listReads; + yield* readAt(306_999); + assert.equal(remote.control.listReads, reads); + remote.control.hideAdmissions = false; + yield* readAt(307_000); + assert.isAbove(remote.control.listReads, reads); + assert.equal((yield* harness.journal.read)[0]?.state, "completed"); + assert.equal(remote.control.preparePosts, 1); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live( + "reports a sent but rejected POST separately from an unsent request across restart", + () => + Effect.gen(function* () { + const remote = yield* fixture; + remote.control.prepareStatus = 400; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const harness = yield* admissionHarness(remote, directory); + const opened = yield* harness.open; + const terminal = yield* Deferred.make(); + yield* opened.runtime.events.pipe( + Stream.runForEach((event) => + event.type === "turn.terminal" ? Deferred.succeed(terminal, event) : Effect.void, + ), + Effect.forkScoped, + ); + yield* opened.runtime.startTurn(harness.turn(opened.thread)); + const event = yield* Deferred.await(terminal); + assert.equal(event.type, "turn.terminal"); + if (event.type === "turn.terminal") { + assert.equal(event.failure?.code, "kilo_cloud_submission_rejected"); + assert.notInclude(event.failure?.message ?? "", "No paid request was sent"); + } + const saved = (yield* harness.journal.read)[0]!; + assert.equal(saved.submissionPhase, "post_attempted"); + assert.isTrue(saved.submissionRejected); + assert.equal(remote.control.preparePosts, 1); + const restored = yield* (yield* admissionHarness(remote, directory)).open; + assert.equal(restored.thread.nativeMetadata?.cloudExecution?.task, "not_started"); + remote.control.prepareStatus = 200; + yield* restored.runtime.startTurn(harness.turn(restored.thread, 2)); + assert.equal(remote.control.preparePosts, 2); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live( + "finishes a durably rejected submission after a crash before terminal save", + () => + Effect.gen(function* () { + const { remote, directory } = yield* uncertainAdmission; + const harness = yield* admissionHarness(remote, directory); + const saved = (yield* harness.journal.read)[0]!; + // Reproduce the durable boundary after a definite rejection, before finish. + yield* harness.journal.save({ ...saved, submissionRejected: true }); + const reads = remote.control.listReads; + const resumed = yield* harness.open; + assert.equal((yield* harness.journal.read)[0]?.state, "failed"); + assert.equal((yield* harness.journal.read)[0]?.submissionPhase, "post_attempted"); + assert.equal(resumed.thread.nativeMetadata?.cloudExecution?.task, "not_started"); + assert.equal(remote.control.listReads, reads); + assert.equal(remote.control.preparePosts, 1); + remote.control.hideAdmissions = false; + yield* resumed.runtime.startTurn(harness.turn(resumed.thread, 2)); + assert.equal(remote.control.preparePosts, 2); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live( + "adopts a terminal preflight Stop from another adapter without rewriting its outcome", + () => + Effect.gen(function* () { + const { remote, directory } = yield* uncertainAdmission; + const harness = yield* admissionHarness(remote, directory); + const opened = yield* harness.open; + const saved = (yield* harness.journal.read)[0]!; + // A concurrent preflight owner may finish Stop before this reader refreshes. + // SQL restores that durable boundary without loosening production phase guards. + const db = yield* Effect.acquireRelease( + Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), + (db) => Effect.sync(() => db.close()), + ); + db.prepare("UPDATE intents SET state = ?, body = ? WHERE operation_key = ?").run( + "interrupted", + JSON.stringify({ + ...saved, + revision: saved.revision + 1, + state: "interrupted", + submissionPhase: "preflight", + }), + saved.operationKey, + ); + const reads = remote.control.listReads; + yield* opened.runtime.readThreadSnapshot({ providerThread: opened.thread }); + assert.equal((yield* harness.journal.read)[0]?.state, "interrupted"); + assert.equal(remote.control.listReads, reads); + assert.isFalse(yield* opened.runtime.hasPendingBackgroundWork!); + assert.equal(remote.control.preparePosts, 1); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts index 5970151aed63..c57d9a103574 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -202,6 +202,8 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { let watching = false; let streamWatching = false; let streamFiber: Fiber.Fiber | undefined; + let admissionStoragePaused = false; + let admissionFailures = 0; let admissionProbeAt = 0; let admissionProbeDelay = 2_000; let streamCursor = 0; @@ -285,7 +287,9 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { active ? options.client.forgetAdmission(options.repository, active.messageId) : Effect.void, ); const hasBackgroundWork = () => - (!!active && !active.admissionRecoveryPaused) || needsHistoryRestore || monitorSandbox; + (!!active && !active.admissionRecoveryPaused && !admissionStoragePaused) || + needsHistoryRestore || + monitorSandbox; const finish = Effect.fn("KiloCloudAdapterV2.finish")(function* ( terminal: "completed" | "failed" | "interrupted", resultFailure?: string, @@ -293,12 +297,14 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { if (!active) return; const at = yield* DateTime.now; const notSubmitted = active.submissionPhase === "preflight"; + const rejected = active.submissionRejected === true; const saved = { ...active, state: terminal, providerTurn: { ...active.providerTurn, status: terminal, completedAt: at }, }; yield* save(saved); + taskState = notSubmitted || rejected ? "not_started" : (saved.remoteState ?? terminal); yield* options.client.forgetAdmission(options.repository, saved.messageId); if (thread?.nativeMetadata?.cloudExecution) { thread = { @@ -308,7 +314,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ...thread.nativeMetadata, cloudExecution: { ...thread.nativeMetadata.cloudExecution, - task: notSubmitted ? "not_started" : (saved.remoteState ?? terminal), + task: taskState, ...(saved.resultStatus ? { result: saved.resultStatus } : {}), }, }, @@ -364,13 +370,17 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { class: "provider_error", code: notSubmitted ? "kilo_cloud_not_submitted" - : resultFailure - ? "kilo_cloud_result_unavailable" - : "provider_error", + : rejected + ? "kilo_cloud_submission_rejected" + : resultFailure + ? "kilo_cloud_result_unavailable" + : "provider_error", ...(resultFailure ? { retryable: false } : {}), message: notSubmitted - ? "Kilo Cloud preflight failed before submission. No paid request was sent; you can try a new turn." - : (resultFailure ?? "Kilo Cloud reported a failed task."), + ? "The request ended before submission. No paid request was sent; you can try a new turn." + : rejected + ? "Kilo Cloud rejected the submission. The request was sent, but no task was accepted. You can try a new turn." + : (resultFailure ?? "Kilo Cloud reported a failed task."), }), threadDisposition: "reusable", } @@ -395,7 +405,6 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { // the thread can retry history independently of the ended turn. needsHistoryRestore = false; if (!binding) monitorSandbox = false; - taskState = notSubmitted ? "not_started" : (saved.remoteState ?? terminal); resultStatus = saved.resultStatus; yield* Deferred.succeed(terminalSignal, undefined); }); @@ -635,70 +644,133 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { requests.set(requestId, { runtime, native, node, item: turnItem }); yield* emit({ type: "turn_item.updated", driver: driver, turnItem }); }); - const reconcile = Effect.fn("KiloCloudAdapterV2.reconcile")(function* () { - if (active?.submissionPhase === "preflight") { - yield* finish("failed"); - return; - } - if (active?.admissionRecoveryPaused) return; - const expectedMessageId = active?.messageId; - if (!binding) { - if (active && !active.prepared) { - const now = yield* Clock.currentTimeMillis; - if (now < admissionProbeAt) return; - admissionProbeAt = now + admissionProbeDelay; - admissionProbeDelay = Math.min(admissionProbeDelay * 2, 60_000); - const found = yield* options.client - .findAdmission(options.repository, active.messageId) - .pipe( - Effect.timeout("8 seconds"), - Effect.catch((cause) => - Effect.gen(function* () { - const failures = (active!.admissionRecoveryFailures ?? 0) + 1; - const paused = - failures >= 3 || - (isCloudError(cause) && - (cause.reason === "recovery_incomplete" || cause.reason === "wrong_owner")); - yield* save({ - ...active!, - admissionRecoveryFailures: failures, - admissionRecoveryPaused: paused, - }); - if (paused) { - monitorSandbox = false; - yield* status( - "Cloud admission recovery is incomplete and automatic scanning is paused. Reopen history to retry reads. Submission and billing remain unknown; do not submit again.", - ); - } - return null; - }), - ), - ); - if (found && active?.messageId === expectedMessageId) - yield* save({ ...active, prepared: found }); + const refreshIntent = Effect.gen(function* () { + if (!active) return; + const latest = (yield* wire(options.journal.readThread(thread!.id))).find( + (entry) => entry.operationKey === active!.operationKey, + ); + if (latest && latest.revision > active.revision) { + active = latest; + if (latest.binding) { + binding = latest.binding; + monitorSandbox = true; } - if (active?.prepared) { - binding = yield* wire( - options.client.bind( - active.prepared, - options.repository, - active.messageId, - options.branch, + } + }); + const finishKnownOutcome = Effect.gen(function* () { + if (!active) return false; + const terminal = active.state; + if (terminal === "completed" || terminal === "failed" || terminal === "interrupted") { + yield* finish( + terminal, + active.remoteState === "completed" && terminal === "failed" + ? "Kilo Cloud completed remotely, but its result was unavailable before the recovery deadline." + : undefined, + ).pipe(Effect.uninterruptible); + return true; + } + if (active.submissionPhase === "preflight" || active.submissionRejected) { + yield* finish("failed").pipe(Effect.uninterruptible); + return true; + } + return false; + }); + const recoverAdmission = Effect.gen(function* () { + yield* refreshIntent; + if (!active || (yield* finishKnownOutcome)) return; + if (active.admissionRecoveryPaused || admissionStoragePaused || binding) return; + const now = yield* Clock.currentTimeMillis; + if (now < admissionProbeAt) return; + admissionProbeAt = now + admissionProbeDelay; + admissionProbeDelay = Math.min(admissionProbeDelay * 2, 60_000); + if (!active.prepared) { + const found = yield* options.client.findAdmission(options.repository, active.messageId); + if (found) yield* save({ ...active, prepared: found }); + } + if (active?.prepared) { + const recovered = yield* wire( + options.client.bind( + active.prepared, + options.repository, + active.messageId, + options.branch, + ), + ); + yield* save({ + ...active, + binding: recovered, + state: "active", + admissionRecoveryFailures: 0, + }); + binding = recovered; + monitorSandbox = true; + yield* options.client.forgetAdmission(options.repository, active!.messageId); + } else if (active?.admissionRecoveryFailures) { + yield* save({ ...active, admissionRecoveryFailures: 0 }); + } + admissionFailures = 0; + }).pipe( + Effect.timeout("8 seconds"), + Effect.catch((cause) => + Effect.gen(function* () { + // Another adapter may have won the CAS. Adopt its progress before + // accounting this failure; never overwrite it with our stale record. + if (!active || binding) return; + admissionFailures = + Math.max(admissionFailures, active.admissionRecoveryFailures ?? 0) + 1; + const paused = + admissionFailures >= 3 || + (isCloudError(cause) && + ["recovery_incomplete", "recovery_limit", "wrong_owner"].includes(cause.reason)); + // Count and pause locally before further I/O: an unavailable journal + // must not let the outer reconciliation timeout erase all progress. + if (paused) { + admissionStoragePaused = true; + monitorSandbox = false; + } + yield* refreshIntent.pipe(Effect.timeout("500 millis"), Effect.ignore); + if (!active || binding) { + admissionStoragePaused = false; + admissionFailures = 0; + return; + } + yield* save({ + ...active, + admissionRecoveryFailures: admissionFailures, + admissionRecoveryPaused: paused, + }).pipe( + Effect.timeout("500 millis"), + Effect.tap(() => + Effect.sync(() => { + admissionStoragePaused = false; + }), + ), + Effect.catch(() => + Effect.sync(() => { + if (paused) admissionStoragePaused = true; + }), ), ); - yield* save({ ...active, binding, state: "active" }); - yield* options.client.forgetAdmission(options.repository, active!.messageId); - } else return; - } + if (paused) { + monitorSandbox = false; + yield* status( + admissionStoragePaused + ? "Cloud recovery is paused because its journal cannot be updated. Restore local storage and reopen history. Submission and billing remain unknown; no request was resubmitted." + : isCloudError(cause) && cause.reason === "recovery_limit" + ? "Cloud recovery reached the 100-page history limit. Reopening repeats this limit; contact support to resolve the existing operation. Submission and billing remain unknown; do not submit again." + : `Cloud admission recovery is incomplete and automatic scanning is paused${isCloudError(cause) ? ` (${cause.recoveryCause ?? cause.reason})` : ""}. Reopen history to retry reads. Submission and billing remain unknown; do not submit again.`, + ); + } + }), + ), + ); + const reconcile = Effect.fn("KiloCloudAdapterV2.reconcile")(function* () { + if (!binding) yield* recoverAdmission; + else yield* refreshIntent; + if (yield* finishKnownOutcome) return; + if (!binding || active?.admissionRecoveryPaused || admissionStoragePaused) return; + const expectedMessageId = active?.messageId; const intents = yield* wire(options.journal.readThread(thread!.id)); - const persisted = intents.find((entry) => entry.messageId === expectedMessageId); - if ( - persisted && - active && - active.messageId === expectedMessageId && - persisted.revision > active.revision - ) - active = persisted; const nowMs = yield* Clock.currentTimeMillis; const previousRecovery = active?.resultRecovery; if (previousRecovery && nowMs < previousRecovery.nextAttemptMs) return; @@ -1085,7 +1157,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { return yield* error("Cloud journal belongs to another account or repository."); binding = last?.binding ?? undefined; taskState = - last?.submissionPhase === "preflight" + last?.submissionPhase === "preflight" || last?.submissionRejected ? "not_started" : (last?.remoteState ?? (last?.state === "awaiting_result" @@ -1103,7 +1175,8 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ? last : undefined; if (binding) thread = { ...thread, nativeThreadRef: nativeRef(binding.kiloSessionId) }; - if (active?.submissionPhase === "preflight") yield* finish("failed"); + if (active?.submissionPhase === "preflight" || active?.submissionRejected) + yield* finish("failed").pipe(Effect.uninterruptible); if (active?.admissionRecoveryPaused) yield* status( "Cloud admission recovery is paused. Reopen history to retry reads; submission and billing remain unknown.", @@ -1279,6 +1352,10 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { return yield* error("A previous cloud admission still needs reconciliation."); resultStatus = undefined; active = intent; + admissionProbeAt = 0; + admissionProbeDelay = 2_000; + admissionFailures = 0; + admissionStoragePaused = false; taskState = "admission_unknown"; monitorSandbox = true; thread = providerThread; @@ -1348,12 +1425,21 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { yield* emit({ type: "provider_thread.updated", driver, providerThread: thread }); }).pipe( Effect.catch((cause) => - active?.submissionPhase === "preflight" || - (!submissionConfirmed && isCloudError(cause) && cause.reason === "rejected") - ? finish("failed") - : status( + Effect.gen(function* () { + if (active?.submissionPhase === "preflight") { + yield* finish("failed").pipe(Effect.uninterruptible); + } else if ( + !submissionConfirmed && + isCloudError(cause) && + cause.reason === "rejected" + ) { + if (active) yield* save({ ...active, submissionRejected: true }); + yield* finish("failed").pipe(Effect.uninterruptible); + } else + yield* status( "Cloud admission is uncertain. Its operation ID is saved; no automatic retry will start another paid task.", - ), + ); + }), ), ); yield* watch; @@ -1424,14 +1510,19 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { gate.withPermit( Effect.gen(function* () { yield* owned(request.providerThread); - const resumeAdmissionWatch = active?.admissionRecoveryPaused === true; + const resumeAdmissionWatch = + active?.admissionRecoveryPaused === true || admissionStoragePaused; if (resumeAdmissionWatch) { + yield* refreshIntent; yield* save({ ...active!, admissionRecoveryPaused: false, admissionRecoveryFailures: 0, }); admissionProbeAt = 0; + admissionProbeDelay = 2_000; + admissionFailures = 0; + admissionStoragePaused = false; } yield* reconcile().pipe( Effect.timeout("10 seconds"), diff --git a/apps/server/src/provider/kilo/KiloCloudClient.ts b/apps/server/src/provider/kilo/KiloCloudClient.ts index a67145b1dca0..1c653a024223 100644 --- a/apps/server/src/provider/kilo/KiloCloudClient.ts +++ b/apps/server/src/provider/kilo/KiloCloudClient.ts @@ -18,7 +18,9 @@ export class KiloCloudError extends Schema.TaggedError()("KiloCl "wrong_owner", "unsupported", "recovery_incomplete", + "recovery_limit", ]), + recoveryCause: Schema.optional(Schema.String), messageId: Schema.optional(Schema.String), }) {} diff --git a/apps/server/src/provider/kilo/KiloCloudJournal.ts b/apps/server/src/provider/kilo/KiloCloudJournal.ts index 9ce1a1a9f473..cba2bd9d0d32 100644 --- a/apps/server/src/provider/kilo/KiloCloudJournal.ts +++ b/apps/server/src/provider/kilo/KiloCloudJournal.ts @@ -57,6 +57,7 @@ export const CloudIntent = Schema.Struct({ incompleteReplySeen: Schema.optional(Schema.Boolean), }), ), + submissionRejected: Schema.optional(Schema.Boolean), admissionRecoveryPaused: Schema.optional(Schema.Boolean), admissionRecoveryFailures: Schema.optional(Schema.Number), interruptRequested: Schema.Boolean, diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts index 8d79dc87fa94..e566b6a67b3a 100644 --- a/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts @@ -8,6 +8,7 @@ import * as Effect from "effect/Effect"; import * as Clock from "effect/Clock"; import * as Redacted from "effect/Redacted"; import * as Cloud from "./KiloCloudWebClient.ts"; +import { KiloCloudError } from "./KiloCloudClient.ts"; const cleanups: Array<() => Promise> = []; afterEach(async () => { @@ -360,7 +361,9 @@ describe("Kilo personal Cloud control-plane customer API", () => { }, [profile], ); - expect((await run(client.prepare(start).pipe(Effect.flip))).reason).toBe("rejected"); + expect((await run(client.prepare(start, Effect.void).pipe(Effect.flip))).reason).toBe( + "rejected", + ); expect(mutations).toBe(0); }); it("fails closed when the repository binding references an unavailable profile", async () => { @@ -373,7 +376,9 @@ describe("Kilo personal Cloud control-plane customer API", () => { [], [{ repoFullName: "FIXTURE/PROJECT", platform: "github", profileId: "unavailable" }], ); - expect((await run(client.prepare(start).pipe(Effect.flip))).reason).toBe("rejected"); + expect((await run(client.prepare(start, Effect.void).pipe(Effect.flip))).reason).toBe( + "rejected", + ); expect(mutations).toBe(0); }); it("uses customer authentication and fixed admission identities, no commits, setup or local data", async () => { @@ -393,7 +398,7 @@ describe("Kilo personal Cloud control-plane customer API", () => { }); }); }); - await run(client.prepare(start)); + await run(client.prepare(start, Effect.void)); expect(bodies).toHaveLength(1); expect(bodies[0]).toMatchObject({ operationKey: start.operationKey, @@ -420,7 +425,7 @@ describe("Kilo personal Cloud control-plane customer API", () => { res.destroy(); }); }); - const error = await run(client.prepare(start).pipe(Effect.flip)); + const error = await run(client.prepare(start, Effect.void).pipe(Effect.flip)); expect(error.reason).toBe("admission_unknown"); expect(error.messageId).toBe(messageId); expect(accepted).toBe(1); @@ -448,7 +453,7 @@ describe("Kilo personal Cloud control-plane customer API", () => { ( await run( client - .send(binding, { messageId, prompt: "Continue", model: "fixture/model" }) + .send(binding, { messageId, prompt: "Continue", model: "fixture/model" }, Effect.void) .pipe(Effect.flip), ) ).reason, @@ -546,3 +551,92 @@ describe("Kilo personal Cloud control-plane customer API", () => { expect((await run(client.billing(binding))).phase).toBe("active"); }); }); + +describe("cloud recovery budgets and paid dispatch boundary", () => { + it("preserves a recently touched scan's candidate budget when the cache fills", async () => { + let lists = 0; + const { client } = await server((request, response) => { + if (request.url?.startsWith("/api/trpc/cliSessionsV2.list")) { + lists++; + return json(response, { + cliSessions: [ + { + session_id: binding.kiloSessionId, + cloud_agent_session_id: binding.cloudAgentSessionId, + }, + ], + nextCursor: null, + }); + } + response.writeHead(503); + response.end(); + }); + await run(client.findAdmission(binding.repository, "hot")); + for (let i = 0; i < 63; i++) await run(client.findAdmission(binding.repository, `cold-${i}`)); + await run(client.findAdmission(binding.repository, "hot")); + await run(client.findAdmission(binding.repository, "new")); + const before = lists; + expect( + (await run(client.findAdmission(binding.repository, "hot").pipe(Effect.flip))).reason, + ).toBe("recovery_incomplete"); + expect(lists).toBe(before); // Kept its original scan, including failed-candidate rounds. + }); + it("distinguishes a finite page limit from transient recovery failures", async () => { + let pages = 0; + const { client } = await server((req, res) => { + expect(req.method).toBe("GET"); + pages++; + json(res, { cliSessions: [], nextCursor: String(pages) }); + }); + for (let i = 0; i < 4; i++) + expect(await run(client.findAdmission(binding.repository, messageId))).toBeNull(); + expect( + (await run(client.findAdmission(binding.repository, messageId).pipe(Effect.flip))).reason, + ).toBe("recovery_limit"); + expect(pages).toBe(101); + }); + it("retains a typed rejection reason when repeated recovery reads are paused", async () => { + const { client } = await server((_req, res) => { + res.writeHead(403); + res.end(); + }); + await run(client.findAdmission(binding.repository, messageId).pipe(Effect.flip)); + await run(client.findAdmission(binding.repository, messageId).pipe(Effect.flip)); + const failure = await run( + client.findAdmission(binding.repository, messageId).pipe(Effect.flip), + ); + expect(failure.reason).toBe("recovery_incomplete"); + expect(failure.recoveryCause).toBe("rejected"); + }); + it("does not execute a paid POST until its caller's durable marker succeeds", async () => { + let posts = 0; + let marked = false; + const { client } = await server((req, res) => { + expect(req.method).toBe("POST"); + expect(marked).toBe(true); + posts++; + json(res, { + cloudAgentSessionId: binding.cloudAgentSessionId, + kiloSessionId: binding.kiloSessionId, + }); + }); + await run( + client + .prepare( + start, + Effect.fail(new KiloCloudError({ operation: "fixture-journal", reason: "rejected" })), + ) + .pipe(Effect.flip), + ); + expect(posts).toBe(0); + await run( + client.prepare( + start, + Effect.sync(() => { + marked = true; + }), + ), + ); + expect(posts).toBe(1); + }); +}); diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.ts index 58f694bd1293..ecfa2df56709 100644 --- a/apps/server/src/provider/kilo/KiloCloudWebClient.ts +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.ts @@ -396,6 +396,8 @@ export const make = (options: { admissionScans.set(key, scan); } scan.touchedAt = now; + admissionScans.delete(key); + admissionScans.set(key, scan); // At most 25 candidate reads per call. Later polls continue this scan. for (let budget = 25; budget > 0; budget--) { if (!scan.pending.length) { @@ -431,11 +433,10 @@ export const make = (options: { nextCursor: Schema.NullOr(Schema.String), }), ); - if ( - page.nextCursor && - (scan.seenCursors.has(page.nextCursor) || scan.seenCursors.size >= 100) - ) + if (page.nextCursor && scan.seenCursors.has(page.nextCursor)) return yield* failure("reconcile-admission", "recovery_incomplete"); + if (page.nextCursor && scan.seenCursors.size >= 100) + return yield* failure("reconcile-admission", "recovery_limit"); if (page.nextCursor) scan.seenCursors.add(page.nextCursor); scan.cursor = page.nextCursor ?? undefined; scan.loaded = true; @@ -481,11 +482,19 @@ export const make = (options: { Effect.gen(function* () { const key = `${repository}\0${initialMessageId}`; const scan = admissionScans.get(key); - if (scan && ++scan.failures >= 3) { + if (scan && ++scan.failures >= 3 && cause.reason !== "recovery_limit") { admissionScans.delete(key); - return yield* failure("reconcile-admission", "recovery_incomplete"); + return yield* new KiloCloudError({ + operation: cause.operation, + reason: "recovery_incomplete", + recoveryCause: cause.recoveryCause ?? cause.reason, + }); } - if (cause.reason === "recovery_incomplete" || cause.reason === "wrong_owner") + if ( + cause.reason === "recovery_incomplete" || + cause.reason === "recovery_limit" || + cause.reason === "wrong_owner" + ) admissionScans.delete(key); return yield* cause; }), @@ -506,7 +515,7 @@ export const make = (options: { readonly model: string; readonly variant?: string; }, - beforePaidPost: Effect.Effect = Effect.void, + beforePaidPost: Effect.Effect, ) => preflight(input.repository).pipe( Effect.andThen( @@ -570,7 +579,7 @@ export const make = (options: { readonly model: string; readonly variant?: string; }, - beforePaidPost: Effect.Effect = Effect.void, + beforePaidPost: Effect.Effect, ) => check(binding).pipe( Effect.andThen( diff --git a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts index bc416ccaf27b..0f586f907918 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts @@ -15,7 +15,6 @@ import * as Scope from "effect/Scope"; import * as Schema from "effect/Schema"; import { describe } from "vite-plus/test"; -import * as ServerLedger from "../OpenCodeServerLedger.ts"; import * as KiloRuntime from "./KiloRuntime.ts"; import { KiloDriver } from "../Drivers/KiloDriver.ts"; import * as ServerConfig from "../../config.ts"; @@ -338,16 +337,13 @@ describe.skipIf(!binary)("KiloRuntime native lifecycle", () => { const replacementProfile = globalLedger ? path.join(root, "moved-profile") : profile; if (globalLedger) { yield* fs.rename(profile, replacementProfile); - // This is the same server-global startup reaper. No account/profile - // lookup is needed, so removal from settings cannot hide the process. - const ledger = yield* ServerLedger.make({ stateDir: processStateDirectory }); - yield* ledger.reapOrphans; yield* fs.makeDirectory(profile); } const restarted = yield* KiloRuntime.make({ instanceId: "crash-fixture", binaryPath: binary!, profileDirectory: replacementProfile, + ...(globalLedger ? { processStateDirectory } : {}), environment: { ...environment, HOME: profile }, }); assert.isFalse(yield* running(message.pid)); diff --git a/apps/server/src/provider/kilo/KiloRuntime.ts b/apps/server/src/provider/kilo/KiloRuntime.ts index 137895ddf06e..912cfdac920d 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.ts @@ -107,13 +107,14 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { const ledger = yield* ServerLedger.make({ stateDir: input.processStateDirectory ?? path.join(profile, "t3-processes"), }); + // Await profile handoff before any caller can spawn a replacement. Removed + // profiles are also covered by OpenCodeServerLedger.layer's boot reaper, + // which shares this stateDir/opencode-servers directory. + yield* ledger.reapOrphans; if (input.processStateDirectory) { - // The server-global ledger also reaps at startup, even when no Kilo profile - // remains configured. Do not block model discovery on orphan shutdown. - yield* ledger.reapOrphans.pipe(Effect.forkIn(owner)); const legacy = yield* ServerLedger.make({ stateDir: path.join(profile, "t3-processes") }); - yield* legacy.reapOrphans.pipe(Effect.forkIn(owner)); - } else yield* ledger.reapOrphans; + yield* legacy.reapOrphans; + } const authContent = input.authContent ?? (yield* readAuth(profile, input.environment)); const environment: NodeJS.ProcessEnv = { ...input.environment, diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index e9c4ce9329ed..f20c2e719454 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -10604,7 +10604,7 @@ export default function ChatView(props: ChatViewProps) { addFolders: (folders) => composerRef.current?.addDroppedFolders(folders), }); - return ( + const workspaceContent = (

); + // Composer chips and panels can render Markdown outside the timeline too. + return ( + + {workspaceContent} + + ); } diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md index ed4790f6ced0..363612d3666a 100644 --- a/docs/user/providers-kilo.md +++ b/docs/user/providers-kilo.md @@ -97,5 +97,9 @@ If preflight fails before the paid request is attempted, T3 ends that turn local and permits an explicit new turn. Interrupted or older admission records without proof of that boundary remain uncertain. A timeout or an incomplete search never permits automatic resubmission. Repeatedly unreadable admission candidates pause -automatic scanning; reopening history retries only reads. Remote task and billing -status remain unknown until Kilo confirms the original operation. +automatic scanning; reopening history retries only reads. A search also stops at +100 history pages. Reopening cannot bypass that limit; resolving such an operation +requires provider support, not another submission. Journal failures pause recovery +in memory if the pause cannot be saved. After storage is repaired or T3 restarts, +the durable uncertain request still prevents a second paid start. Remote task and +billing status remain unknown until Kilo confirms the original operation. From d7db2431fdcac224da386f7b7c7c0125a28267ec Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sun, 4 Oct 2026 10:36:01 +0000 Subject: [PATCH 19/44] test(kilo): encode recovery fixture with its journal schema --- .../src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index 23adab450775..1cf6947f468a 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -54,6 +54,9 @@ const clockAt = (clock: Clock.Clock, millis: number): Clock.Clock => ({ sleep: clock.sleep.bind(clock), }); const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); +const encodeIntent = Schema.encodeEffect( + Schema.fromJsonString(Schema.toCodecJson(Journal.CloudIntent)), +); const fixture = Effect.acquireRelease( Effect.promise(async () => { let submissions = 0; @@ -1985,7 +1988,7 @@ it.live( ); db.prepare("UPDATE intents SET state = ?, body = ? WHERE operation_key = ?").run( "interrupted", - JSON.stringify({ + yield* encodeIntent({ ...saved, revision: saved.revision + 1, state: "interrupted", From 1baf9ad1cd3342dfa6b910cbfee2f3d28eb7b541 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sun, 4 Oct 2026 14:29:13 +0000 Subject: [PATCH 20/44] fix(kilo): preserve cloud outcomes across storage failures --- .../src/components/LocalWorkspaceNotice.tsx | 11 +- .../src/features/review/ReviewSheet.tsx | 2 +- .../Adapters/KiloCloudAdapterV2.test.ts | 270 +++++++++++++++++- .../Adapters/KiloCloudAdapterV2.ts | 105 +++++-- .../src/provider/OpenCodeServerLedger.test.ts | 67 +++++ .../src/provider/OpenCodeServerLedger.ts | 31 +- .../provider/kilo/KiloCloudWebClient.test.ts | 33 +++ .../src/provider/kilo/KiloCloudWebClient.ts | 6 + apps/web/src/components/ChatView.tsx | 192 ++++++------- docs/user/providers-kilo.md | 6 + 10 files changed, 598 insertions(+), 125 deletions(-) diff --git a/apps/mobile/src/components/LocalWorkspaceNotice.tsx b/apps/mobile/src/components/LocalWorkspaceNotice.tsx index dd67144995b2..7bbb4d78a48d 100644 --- a/apps/mobile/src/components/LocalWorkspaceNotice.tsx +++ b/apps/mobile/src/components/LocalWorkspaceNotice.tsx @@ -1,7 +1,7 @@ import { NativeStackScreenOptions } from "../native/StackHeader"; -import { View } from "react-native"; +import { ActivityIndicator, View } from "react-native"; import { EmptyState } from "./EmptyState"; -import { LoadingScreen } from "./LoadingScreen"; +import { AppText } from "./AppText"; import type { threadLocalWorkspace } from "../state/threadLocalWorkspace"; export function LocalWorkspaceNotice({ @@ -15,7 +15,12 @@ export function LocalWorkspaceNotice({ {state === "loading" ? ( - + + + + Loading conversation workspace... + + ) : ( ; + return ; return ; } diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index 1cf6947f468a..f784adb000fe 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -89,17 +89,25 @@ const fixture = Effect.acquireRelease( }); const control = { prepareStatus: 200, + parkPrepare: false, + parkedPrepare: undefined as NodeHttp.ServerResponse | undefined, + prepareSeen: undefined as (() => void) | undefined, preparePosts: 0, listReads: 0, listStatus: 200, parkList: false, + parkedList: undefined as NodeHttp.ServerResponse | undefined, listSeen: undefined as (() => void) | undefined, listClosed: undefined as (() => void) | undefined, resultReads: 0, completeAfterResultReads: 0, sessionStatus: 200, + sessionBranch: "main", malformedSession: false, sendPosts: 0, + parkSend: false, + parkedSend: undefined as NodeHttp.ServerResponse | undefined, + sendSeen: undefined as (() => void) | undefined, profileStatus: 200, personalAccount: true, profileAccount: "fixture-account", @@ -199,6 +207,15 @@ const fixture = Effect.acquireRelease( } if (operation === "cloudAgentNext.prepareSession") { control.preparePosts++; + if (control.parkPrepare) { + control.parkedPrepare = response; + response.once("close", () => { + if (control.parkedPrepare === response) control.parkedPrepare = undefined; + }); + response.on("error", () => {}); + control.prepareSeen?.(); + return; + } if (control.prepareStatus !== 200) { response.writeHead(control.prepareStatus); response.end(); @@ -224,7 +241,11 @@ const fixture = Effect.acquireRelease( } if (operation === "cliSessionsV2.list") control.listReads++; if (operation === "cliSessionsV2.list" && control.parkList) { - response.once("close", () => control.listClosed?.()); + control.parkedList = response; + response.once("close", () => { + if (control.parkedList === response) control.parkedList = undefined; + control.listClosed?.(); + }); response.on("error", () => {}); control.listSeen?.(); return; @@ -282,13 +303,22 @@ const fixture = Effect.acquireRelease( worktreeId: state.worktree, userId: "fixture-account", githubRepo: "fixture/repo", - upstreamBranch: "main", + upstreamBranch: control.sessionBranch, autoCommit: false, initialMessageId: state.initial, execution: null, }); if (operation === "cloudAgentNext.sendMessage") { control.sendPosts++; + if (control.parkSend) { + control.parkedSend = response; + response.once("close", () => { + control.parkedSend = undefined; + }); + response.on("error", () => {}); + control.sendSeen?.(); + return; + } const payload = input.payload as unknown as { prompt: string }; state.messages.push({ id: input.messageId!, prompt: payload.prompt }); return reply({ @@ -1826,6 +1856,11 @@ it.live( const restarted = yield* admissionHarness(remote, directory); const next = yield* restarted.open; yield* next.runtime.startTurn(restarted.turn(next.thread, 2)).pipe(Effect.flip); + const paused = yield* opened.runtime + .readThreadSnapshot({ providerThread: opened.thread }) + .pipe(Effect.flip); + assert.include(paused.message, "Cloud recovery remains paused"); + assert.equal(remote.control.preparePosts, 1); db.exec("DROP TRIGGER fail_recovery_save"); remote.control.listStatus = 200; remote.control.hideAdmissions = false; @@ -2005,3 +2040,234 @@ it.live( }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), 20_000, ); + +it.live.each(["pause", "terminal"] as const)( + "preserves a concurrent %s after a failed admission probe", + (outcome) => + Effect.gen(function* () { + const { remote, directory } = yield* uncertainAdmission; + const harness = yield* admissionHarness(remote, directory); + const initial = (yield* harness.journal.read)[0]!; + yield* harness.journal.save({ + ...initial, + admissionRecoveryFailures: outcome === "terminal" ? 2 : 0, + }); + const opened = yield* harness.open; + const db = yield* Effect.acquireRelease( + Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), + (db) => Effect.sync(() => db.close()), + ); + if (outcome === "terminal") + db.exec( + "CREATE TRIGGER reject_pause BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.admissionRecoveryPaused') = 1 BEGIN SELECT RAISE(FAIL, 'fixture pause write failure'); END", + ); + remote.control.parkList = true; + const seen = yield* Deferred.make(); + remote.control.listSeen = () => Deferred.doneUnsafe(seen, Effect.void); + const reading = yield* opened.runtime + .readThreadSnapshot({ providerThread: opened.thread }) + .pipe(Effect.forkScoped); + yield* Deferred.await(seen); + const other = yield* Journal.make(`${directory}/journal`); + const latest = (yield* other.read)[0]!; + yield* other.save( + outcome === "pause" + ? { ...latest, admissionRecoveryFailures: 9, admissionRecoveryPaused: true } + : { + ...latest, + state: "interrupted", + providerTurn: { + ...latest.providerTurn, + status: "interrupted", + completedAt: yield* DateTime.now, + }, + }, + ); + remote.control.parkList = false; + remote.control.parkedList!.writeHead(503); + remote.control.parkedList!.end(); + yield* Fiber.join(reading); + const final = (yield* harness.journal.read)[0]!; + if (outcome === "pause") { + assert.isTrue(final.admissionRecoveryPaused); + assert.isAtLeast(final.admissionRecoveryFailures!, 9); + } else { + assert.equal(final.state, "interrupted"); + // Finishing the adopted turn must retire a failed local pause, too. + yield* opened.runtime.readThreadSnapshot({ providerThread: opened.thread }); + assert.equal((yield* harness.journal.read)[0]!.state, "interrupted"); + } + assert.isFalse(yield* opened.runtime.hasPendingBackgroundWork!); + assert.equal(remote.control.preparePosts, 1); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live.each(["storage", "revision"] as const)( + "retains a definite rejection across a %s write failure without resubmitting", + (failure) => + Effect.gen(function* () { + const remote = yield* fixture; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const harness = yield* admissionHarness(remote, directory); + const opened = yield* harness.open; + const db = yield* Effect.acquireRelease( + Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), + (db) => Effect.sync(() => db.close()), + ); + if (failure === "storage") + db.exec( + "CREATE TRIGGER reject_outcome BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN SELECT RAISE(FAIL, 'fixture rejection write failure'); END", + ); + remote.control.parkPrepare = true; + const seen = yield* Deferred.make(); + remote.control.prepareSeen = () => Deferred.doneUnsafe(seen, Effect.void); + const starting = yield* opened.runtime + .startTurn(harness.turn(opened.thread)) + .pipe(Effect.forkScoped); + yield* Deferred.await(seen); + if (failure === "revision") { + const other = yield* Journal.make(`${directory}/journal`); + yield* other.save({ ...(yield* other.read)[0]!, admissionRecoveryFailures: 1 }); + } + remote.control.parkedPrepare!.writeHead(400); + remote.control.parkedPrepare!.end(); + yield* Fiber.join(starting); + assert.equal(remote.control.preparePosts, 1); + assert.equal(remote.control.listReads, 0); + if (failure === "storage") { + assert.isFalse(yield* opened.runtime.hasPendingBackgroundWork!); + const saved = (yield* harness.journal.read)[0]!; + assert.equal(saved.state, "admission_unknown"); + assert.equal(saved.submissionPhase, "post_attempted"); + yield* opened.runtime.startTurn(harness.turn(opened.thread, 2)).pipe(Effect.flip); + // Restart cannot invent the rejection if no outcome could reach disk. + const restarted = yield* admissionHarness(remote, directory); + const next = yield* restarted.open; + yield* next.runtime.startTurn(restarted.turn(next.thread, 2)).pipe(Effect.flip); + yield* next.runtime + .interruptTurn({ providerThread: next.thread, providerTurnId: saved.providerTurn.id }) + .pipe(Effect.flip); + db.exec("DROP TRIGGER reject_outcome"); + yield* opened.runtime.readThreadSnapshot({ providerThread: opened.thread }); + } + const saved = (yield* harness.journal.read)[0]!; + assert.equal(saved.state, "failed"); + assert.isTrue(saved.submissionRejected); + assert.equal(remote.control.listReads, 0); + remote.control.parkPrepare = false; + yield* opened.runtime.startTurn(harness.turn(opened.thread, 2)); + assert.equal(remote.control.preparePosts, 2); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live( + "pauses immediately when recovered admission has the wrong branch", + () => + Effect.gen(function* () { + const { remote, directory } = yield* uncertainAdmission; + remote.control.hideAdmissions = false; + remote.control.sessionBranch = "other-branch"; + const harness = yield* admissionHarness(remote, directory); + const opened = yield* harness.open; + yield* opened.runtime.readThreadSnapshot({ providerThread: opened.thread }); + const saved = (yield* harness.journal.read)[0]!; + assert.isTrue(saved.admissionRecoveryPaused); + assert.equal(saved.admissionRecoveryFailures, 1); + assert.equal(saved.state, "admission_unknown"); + assert.isNull(saved.binding); + assert.isFalse(yield* opened.runtime.hasPendingBackgroundWork!); + assert.equal(remote.control.preparePosts, 1); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live.each(["accepted", "storage", "storage-accepted"] as const)( + "keeps a rejected follow-up safe during concurrent %s handling", + (mode) => + Effect.gen(function* () { + const remote = yield* fixture; + remote.control.status = "failed"; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const harness = yield* admissionHarness(remote, directory); + const firstScope = yield* Scope.fork(yield* Effect.scope); + const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, firstScope)); + const done = yield* Deferred.make(); + yield* first.runtime.events.pipe( + Stream.runForEach((event) => + event.type === "turn.terminal" ? Deferred.succeed(done, undefined) : Effect.void, + ), + Effect.forkIn(firstScope), + ); + yield* first.runtime.startTurn(harness.turn(first.thread)); + yield* Deferred.await(done); + yield* Scope.close(firstScope, Exit.void); + const originalBinding = (yield* harness.journal.read)[0]!.binding; + remote.control.status = "running"; + remote.control.parkSend = true; + const second = yield* harness.open; + const db = yield* Effect.acquireRelease( + Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), + (db) => Effect.sync(() => db.close()), + ); + if (mode !== "accepted") + db.exec( + "CREATE TRIGGER reject_followup BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN SELECT RAISE(FAIL, 'fixture rejection write failure'); END", + ); + const seen = yield* Deferred.make(); + remote.control.sendSeen = () => Deferred.doneUnsafe(seen, Effect.void); + const starting = yield* second.runtime + .startTurn(harness.turn(second.thread, 2)) + .pipe(Effect.forkScoped); + yield* Deferred.await(seen); + const other = yield* Journal.make(`${directory}/journal`); + if (mode === "accepted") + yield* other.save({ ...(yield* other.read)[1]!, remoteState: "running" }); + remote.control.parkedSend!.writeHead(400); + remote.control.parkedSend!.end(); + yield* Fiber.join(starting); + const saved = (yield* other.read)[1]!; + assert.equal(saved.state, "admission_unknown"); + assert.isNotTrue(saved.submissionRejected); + assert.deepEqual(saved.binding, originalBinding); + yield* second.runtime.startTurn(harness.turn(second.thread, 3)).pipe(Effect.flip); + if (mode !== "accepted") { + const stopped = yield* second.runtime + .interruptTurn({ providerThread: second.thread, providerTurnId: saved.providerTurn.id }) + .pipe(Effect.flip); + assert.include(stopped.message, "No remote interrupt was sent"); + assert.equal(remote.control.interruptPosts, 0); + db.exec("DROP TRIGGER reject_followup"); + if (mode === "storage-accepted") { + yield* other.save({ ...(yield* other.read)[1]!, remoteState: "running" }); + remote.control.interruptAccepted = true; + } + yield* second.runtime.interruptTurn({ + providerThread: second.thread, + providerTurnId: saved.providerTurn.id, + }); + const final = (yield* other.read)[1]!; + assert.equal(final.state, mode === "storage" ? "failed" : "interrupted"); + assert.equal(final.submissionRejected === true, mode === "storage"); + assert.equal(remote.control.interruptPosts, mode === "storage" ? 0 : 1); + } else { + assert.equal(saved.remoteState, "running"); + } + assert.equal(remote.control.preparePosts, 1); + assert.equal(remote.control.sendPosts, 1); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts index c57d9a103574..1fda3337c8b8 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -203,6 +203,8 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { let streamWatching = false; let streamFiber: Fiber.Fiber | undefined; let admissionStoragePaused = false; + // Ephemeral proof is never reconstructed from an uncertain persisted row. + let rejectedOperationKey: string | undefined; let admissionFailures = 0; let admissionProbeAt = 0; let admissionProbeDelay = 2_000; @@ -401,6 +403,9 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { providerSession: { ...session, status: "ready", updatedAt: at, lastError: null }, }); active = undefined; + admissionStoragePaused = false; + admissionFailures = 0; + rejectedOperationKey = undefined; // Terminal task state must not pin a history retry forever. Reopening // the thread can retry history independently of the ended turn. needsHistoryRestore = false; @@ -669,6 +674,12 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ).pipe(Effect.uninterruptible); return true; } + if (rejectedOperationKey === active.operationKey) { + // Newer durable acceptance/terminal state outranks an older local outcome. + if (active.state === "admission_unknown" && !active.prepared && !active.remoteState) + active = { ...active, submissionRejected: true }; + else rejectedOperationKey = undefined; + } if (active.submissionPhase === "preflight" || active.submissionRejected) { yield* finish("failed").pipe(Effect.uninterruptible); return true; @@ -688,13 +699,11 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { if (found) yield* save({ ...active, prepared: found }); } if (active?.prepared) { - const recovered = yield* wire( - options.client.bind( - active.prepared, - options.repository, - active.messageId, - options.branch, - ), + const recovered = yield* options.client.bind( + active.prepared, + options.repository, + active.messageId, + options.branch, ); yield* save({ ...active, @@ -718,7 +727,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { if (!active || binding) return; admissionFailures = Math.max(admissionFailures, active.admissionRecoveryFailures ?? 0) + 1; - const paused = + let paused = admissionFailures >= 3 || (isCloudError(cause) && ["recovery_incomplete", "recovery_limit", "wrong_owner"].includes(cause.reason)); @@ -734,6 +743,10 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { admissionFailures = 0; return; } + // A concurrent observer may have already exhausted its scan budget. + // Only an explicit resume can clear that newer durable pause. + admissionFailures = Math.max(admissionFailures, active.admissionRecoveryFailures ?? 0); + paused ||= active.admissionRecoveryPaused === true || admissionFailures >= 3; yield* save({ ...active, admissionRecoveryFailures: admissionFailures, @@ -1433,8 +1446,26 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { isCloudError(cause) && cause.reason === "rejected" ) { - if (active) yield* save({ ...active, submissionRejected: true }); - yield* finish("failed").pipe(Effect.uninterruptible); + rejectedOperationKey = active?.operationKey; + // Commit the rejection and terminal state together. Refresh on + // a CAS conflict, but never retry the paid request. If storage + // stays unavailable, retain the outcome only in this adapter. + yield* Effect.gen(function* () { + yield* refreshIntent; + yield* finishKnownOutcome; + }).pipe( + Effect.retry({ times: 1 }), + Effect.timeout("8 seconds"), + Effect.catch(() => + Effect.gen(function* () { + admissionStoragePaused = true; + monitorSandbox = false; + yield* status( + "Kilo Cloud rejected this request, but the outcome could not be saved. Restore local storage and reopen history. The reservation remains held; no request was resubmitted.", + ); + }), + ), + ); } else yield* status( "Cloud admission is uncertain. Its operation ID is saved; no automatic retry will start another paid task.", @@ -1451,6 +1482,20 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { Effect.gen(function* () { yield* owned(request.providerThread); if (!active || active.providerTurn.id !== request.providerTurnId) return false; + if (rejectedOperationKey === active.operationKey || active.submissionRejected) { + const finished = yield* Effect.gen(function* () { + yield* refreshIntent; + return yield* finishKnownOutcome; + }).pipe( + Effect.timeout("8 seconds"), + Effect.mapError(() => + error( + "Kilo Cloud rejected this request, but its outcome cannot be saved. Restore local storage and reopen history. No remote interrupt was sent.", + ), + ), + ); + if (finished || !active) return false; + } if (active.submissionPhase === "preflight") { yield* finish("interrupted"); return false; @@ -1470,6 +1515,11 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { yield* watch; if (active.interruptRequested) return true; yield* save({ ...active, interruptRequested: true }); + // A successful write retires the local storage pause so this + // explicit Stop can observe remote confirmation. Keep durable + // admission pauses intact. + admissionStoragePaused = false; + admissionFailures = 0; const accepted = yield* wire( options.client .interrupt(binding) @@ -1513,16 +1563,31 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { const resumeAdmissionWatch = active?.admissionRecoveryPaused === true || admissionStoragePaused; if (resumeAdmissionWatch) { - yield* refreshIntent; - yield* save({ - ...active!, - admissionRecoveryPaused: false, - admissionRecoveryFailures: 0, - }); - admissionProbeAt = 0; - admissionProbeDelay = 2_000; - admissionFailures = 0; - admissionStoragePaused = false; + yield* Effect.gen(function* () { + yield* refreshIntent; + if ( + active && + (active.state === "admission_unknown" || + active.state === "active" || + active.state === "awaiting_result") + ) + yield* save({ + ...active, + admissionRecoveryPaused: false, + admissionRecoveryFailures: 0, + }); + admissionProbeAt = 0; + admissionProbeDelay = 2_000; + admissionFailures = 0; + admissionStoragePaused = false; + }).pipe( + Effect.timeout("8 seconds"), + Effect.mapError(() => + error( + "Cloud recovery remains paused because its journal cannot be updated. Restore local storage and reopen history. No request was resubmitted.", + ), + ), + ); } yield* reconcile().pipe( Effect.timeout("10 seconds"), diff --git a/apps/server/src/provider/OpenCodeServerLedger.test.ts b/apps/server/src/provider/OpenCodeServerLedger.test.ts index 99058ab8fe14..c2beb3ca26e0 100644 --- a/apps/server/src/provider/OpenCodeServerLedger.test.ts +++ b/apps/server/src/provider/OpenCodeServerLedger.test.ts @@ -242,3 +242,70 @@ describe.skipIf(observedPlatforms.length === 0)("OpenCode server startup", () => }).pipe(Effect.provide(NodeServices.layer)), ); }); + +it.live.skipIf(hostPlatform !== "linux")( + "serializes boot and profile reapers until a TERM-resistant group has stopped", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ledger-overlap-" }); + const group = yield* Effect.acquireRelease( + Effect.gen(function* () { + const ready = yield* Deferred.make(); + const exited = yield* Deferred.make(); + const outputClosed = yield* Deferred.make(); + const leader = NodeChildProcess.spawn( + process.execPath, + [ + "-e", + ` + const {spawn}=require('node:child_process'); + const child=spawn(process.execPath,['-e',"process.on('SIGTERM',()=>{});process.send(process.pid);setInterval(()=>{},1000)"],{stdio:['ignore',process.stdout,'ignore','ipc']}); + child.once('message',pid=>process.send(pid)); + setInterval(()=>{},1000); + `, + ], + { detached: true, stdio: ["ignore", "pipe", "ignore", "ipc"] }, + ); + leader.once("message", (pid) => Deferred.doneUnsafe(ready, Effect.succeed(Number(pid)))); + leader.once("exit", () => Deferred.doneUnsafe(exited, Effect.void)); + leader.stdout!.once("end", () => Deferred.doneUnsafe(outputClosed, Effect.void)); + leader.stdout!.resume(); + return { + pid: leader.pid!, + ready: Deferred.await(ready), + exited: Deferred.await(exited), + outputClosed: Deferred.await(outputClosed), + }; + }), + (group) => killGroup(group.pid), + ); + const member = yield* group.ready; + yield* recordFromDeadServer(stateDir, group); + const boot = yield* OpenCodeServerLedger.make({ stateDir }); + const profile = yield* OpenCodeServerLedger.make({ stateDir }); + const complete = yield* Deferred.make(); + const first = yield* boot.reapOrphans.pipe( + Effect.andThen(Deferred.succeed(complete, undefined)), + Effect.forkScoped, + ); + yield* group.exited; // TERM killed the recorded leader; its child ignores TERM. + const running = fs.readFileString(`/proc/${member}/stat`).pipe( + Effect.map((stat) => !stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")), + Effect.orElseSucceed(() => false), + ); + expect(yield* running).toBe(true); + yield* profile.reapOrphans; + expect(yield* Deferred.isDone(complete)).toBe(true); + expect(yield* running).toBe(false); + yield* group.outputClosed; + yield* Fiber.join(first); + // Coordination is released after the scan, not cached forever. + const later = yield* spawnGroup(SERVE_ARGS); + yield* recordFromDeadServer(stateDir, later); + yield* profile.reapOrphans; + yield* later.exited; + expect(groupExists(later.pid)).toBe(false); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + 20_000, +); diff --git a/apps/server/src/provider/OpenCodeServerLedger.ts b/apps/server/src/provider/OpenCodeServerLedger.ts index 654e4c0c2889..702506702a5f 100644 --- a/apps/server/src/provider/OpenCodeServerLedger.ts +++ b/apps/server/src/provider/OpenCodeServerLedger.ts @@ -6,11 +6,17 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import * as ServerConfig from "../config.ts"; import { signalProcessGroup } from "../process/processGroup.ts"; +// Boot and provider runtimes create separate ledger instances in one T3 process. +// Serialize their scans, including the process-group grace period. A second scan +// must still run afterward to cover entries created since the first scan began. +const reapers = new Map(); + const ProcessIdentity = Schema.Struct({ pid: Schema.Int, startTime: Schema.String }); type ProcessIdentity = typeof ProcessIdentity.Type; @@ -303,7 +309,7 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { ); /** Stops recorded servers whose owning T3 server is gone and drops stale entries. */ - const reapOrphans = Effect.gen(function* () { + const reapOnce = Effect.gen(function* () { const names = yield* fs.readDirectory(directory).pipe(Effect.orElseSucceed(() => [])); yield* Effect.forEach( names.filter((name) => ENTRY_FILE.test(name)), @@ -312,6 +318,29 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { ); }); + const reapOrphans = Effect.gen(function* () { + const resolved = path.resolve(input.stateDir); + const key = yield* fs.realPath(resolved).pipe(Effect.orElseSucceed(() => resolved)); + yield* Effect.acquireUseRelease( + Effect.sync(() => { + let entry = reapers.get(key); + if (!entry) { + entry = { gate: Semaphore.makeUnsafe(1), users: 0 }; + reapers.set(key, entry); + } + entry.users++; + return entry; + }), + // Once we signal a verified group, cancellation must not release the gate + // before its bounded TERM/KILL cleanup finishes. Waiting callers can cancel. + (entry) => entry.gate.withPermit(reapOnce.pipe(Effect.uninterruptible)), + (entry) => + Effect.sync(() => { + if (--entry.users === 0) reapers.delete(key); + }), + ); + }); + return { track, reapOrphans }; }); diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts index e566b6a67b3a..7ab1e9bbfe84 100644 --- a/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts @@ -640,3 +640,36 @@ describe("cloud recovery budgets and paid dispatch boundary", () => { expect(posts).toBe(1); }); }); + +it("resets transient failures after successful progress in the same incomplete scan", async () => { + let unavailable = true; + let pages = 0; + const { client } = await server((req, res) => { + expect(req.method).toBe("GET"); + if (unavailable) { + res.writeHead(503); + res.end(); + return; + } + pages++; + json(res, { cliSessions: [], nextCursor: String(pages) }); + }); + for (let i = 0; i < 2; i++) + expect( + (await run(client.findAdmission(binding.repository, messageId).pipe(Effect.flip))).reason, + ).toBe("invalid_response"); + unavailable = false; + expect(await run(client.findAdmission(binding.repository, messageId))).toBeNull(); + expect(pages).toBe(25); + unavailable = true; + expect( + (await run(client.findAdmission(binding.repository, messageId).pipe(Effect.flip))).reason, + ).toBe("invalid_response"); + // Progress resets only transient failures, not the finite pagination budget. + unavailable = false; + for (let i = 0; i < 3; i++) await run(client.findAdmission(binding.repository, messageId)); + expect( + (await run(client.findAdmission(binding.repository, messageId).pipe(Effect.flip))).reason, + ).toBe("recovery_limit"); + expect(pages).toBe(101); +}); diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.ts index ecfa2df56709..65e6589dafe0 100644 --- a/apps/server/src/provider/kilo/KiloCloudWebClient.ts +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.ts @@ -478,6 +478,12 @@ export const make = (options: { } return null; }).pipe( + Effect.tap(() => + Effect.sync(() => { + const scan = admissionScans.get(`${repository}\0${initialMessageId}`); + if (scan) scan.failures = 0; + }), + ), Effect.catchTag("KiloCloudError", (cause) => Effect.gen(function* () { const key = `${repository}\0${initialMessageId}`; diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index f20c2e719454..8c9daa17b86e 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -10763,104 +10763,100 @@ export default function ChatView(props: ChatViewProps) { {/* Messages Wrapper */}
{/* Messages — LegendList handles virtualization and scrolling internally */} - - {} : onForkFromRun} - onRollbackCheckpoint={(input) => { - if (!paintOnlyDisplayedTimeline) void onRollbackCheckpoint(input); - }} - supportsConversationRollback={ - !paintOnlyDisplayedTimeline && supportsConversationRollback - } - onRevertToTurnCount={ - paintOnlyDisplayedTimeline ? noopHeldRevert : onRevertTimelineTurn - } - {...(!paintOnlyDisplayedTimeline - ? { onUseArtifactTemplate: useArtifactTemplate } - : {})} - isRevertingCheckpoint={isRevertingCheckpoint} - onImageExpand={onExpandTimelineImage} - onFileOpen={paintOnlyDisplayedTimeline ? noopHeldAttachment : openFileAttachment} - onFileDownload={ - paintOnlyDisplayedTimeline ? noopHeldAttachment : downloadFileAttachment - } - markdownCwd={ - paintOnlyDisplayedTimeline - ? (heldPaintContext?.markdownCwd ?? undefined) - : (gitCwd ?? undefined) - } - resolvedTheme={resolvedTheme} - timestampFormat={timestampFormat} - workspaceRoot={ - paintOnlyDisplayedTimeline - ? (heldPaintContext?.workspaceRoot ?? undefined) - : activeWorkspaceRoot - } - skills={ - activeProviderStatus - ? resolveProviderSkillsForCwd(activeProviderStatus, gitCwd) - : EMPTY_PROVIDER_SKILLS - } - anchorMessageId={paintOnlyDisplayedTimeline ? null : timelineAnchorMessageId} - onAnchorReady={onTimelineAnchorReady} - onAnchorSizeChanged={onTimelineAnchorSizeChanged} - contentInsetEndAdjustment={composerTimelineInset} - liveFollowEnabled={!paintOnlyDisplayedTimeline && timelineLiveFollowEnabled} - onIsAtEndChange={onIsAtEndChange} - onContentOverflowChange={setTimelineOverflows} - onToolOutputCollapsedAtEnd={onToolOutputCollapsedAtEnd} - onManualNavigation={cancelTimelineLiveFollowForUserNavigation} - cancelPositionRestoreRef={cancelPositionRestoreRef} - hideEmptyPlaceholder={isDraftHeroState || threadDetailLoading} - topFadeEnabled={!hasTimelineTopBanner} - {...(paintOnlyDisplayedTimeline || threadHistoryControls === undefined - ? {} - : { historyControls: threadHistoryControls })} - /> - + {} : onForkFromRun} + onRollbackCheckpoint={(input) => { + if (!paintOnlyDisplayedTimeline) void onRollbackCheckpoint(input); + }} + supportsConversationRollback={ + !paintOnlyDisplayedTimeline && supportsConversationRollback + } + onRevertToTurnCount={ + paintOnlyDisplayedTimeline ? noopHeldRevert : onRevertTimelineTurn + } + {...(!paintOnlyDisplayedTimeline + ? { onUseArtifactTemplate: useArtifactTemplate } + : {})} + isRevertingCheckpoint={isRevertingCheckpoint} + onImageExpand={onExpandTimelineImage} + onFileOpen={paintOnlyDisplayedTimeline ? noopHeldAttachment : openFileAttachment} + onFileDownload={ + paintOnlyDisplayedTimeline ? noopHeldAttachment : downloadFileAttachment + } + markdownCwd={ + paintOnlyDisplayedTimeline + ? (heldPaintContext?.markdownCwd ?? undefined) + : (gitCwd ?? undefined) + } + resolvedTheme={resolvedTheme} + timestampFormat={timestampFormat} + workspaceRoot={ + paintOnlyDisplayedTimeline + ? (heldPaintContext?.workspaceRoot ?? undefined) + : activeWorkspaceRoot + } + skills={ + activeProviderStatus + ? resolveProviderSkillsForCwd(activeProviderStatus, gitCwd) + : EMPTY_PROVIDER_SKILLS + } + anchorMessageId={paintOnlyDisplayedTimeline ? null : timelineAnchorMessageId} + onAnchorReady={onTimelineAnchorReady} + onAnchorSizeChanged={onTimelineAnchorSizeChanged} + contentInsetEndAdjustment={composerTimelineInset} + liveFollowEnabled={!paintOnlyDisplayedTimeline && timelineLiveFollowEnabled} + onIsAtEndChange={onIsAtEndChange} + onContentOverflowChange={setTimelineOverflows} + onToolOutputCollapsedAtEnd={onToolOutputCollapsedAtEnd} + onManualNavigation={cancelTimelineLiveFollowForUserNavigation} + cancelPositionRestoreRef={cancelPositionRestoreRef} + hideEmptyPlaceholder={isDraftHeroState || threadDetailLoading} + topFadeEnabled={!hasTimelineTopBanner} + {...(paintOnlyDisplayedTimeline || threadHistoryControls === undefined + ? {} + : { historyControls: threadHistoryControls })} + /> {/* scroll to end pill — shown when user has scrolled away from the live edge */} {showScrollToBottom && ( diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md index 363612d3666a..8a92ff2a7d9f 100644 --- a/docs/user/providers-kilo.md +++ b/docs/user/providers-kilo.md @@ -103,3 +103,9 @@ requires provider support, not another submission. Journal failures pause recove in memory if the pause cannot be saved. After storage is repaired or T3 restarts, the durable uncertain request still prevents a second paid start. Remote task and billing status remain unknown until Kilo confirms the original operation. + +If Kilo explicitly rejects a submitted request while local storage is unavailable, +T3 holds the reservation and keeps that rejection in memory until it can save the +outcome. Reopen history after repairing storage. If T3 exits before that save, the +journal cannot prove the rejection and the request remains uncertain. Stop does +not send a remote interrupt for a known rejected request. From 894f31b6bb2ba8cb1dd26cf1c08708d913df5baa Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sun, 4 Oct 2026 14:32:22 +0000 Subject: [PATCH 21/44] fix(kilo): observe a concurrently persisted cloud stop --- .../Adapters/KiloCloudAdapterV2.test.ts | 13 +++++++++---- .../orchestration-v2/Adapters/KiloCloudAdapterV2.ts | 8 +++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index f784adb000fe..6672ea5f7a46 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -2189,7 +2189,7 @@ it.live( 20_000, ); -it.live.each(["accepted", "storage", "storage-accepted"] as const)( +it.live.each(["accepted", "storage", "storage-accepted", "storage-interrupted"] as const)( "keeps a rejected follow-up safe during concurrent %s handling", (mode) => Effect.gen(function* () { @@ -2251,9 +2251,14 @@ it.live.each(["accepted", "storage", "storage-accepted"] as const)( assert.include(stopped.message, "No remote interrupt was sent"); assert.equal(remote.control.interruptPosts, 0); db.exec("DROP TRIGGER reject_followup"); - if (mode === "storage-accepted") { - yield* other.save({ ...(yield* other.read)[1]!, remoteState: "running" }); + if (mode === "storage-accepted" || mode === "storage-interrupted") { + yield* other.save({ + ...(yield* other.read)[1]!, + remoteState: "running", + interruptRequested: mode === "storage-interrupted", + }); remote.control.interruptAccepted = true; + if (mode === "storage-interrupted") remote.control.status = "interrupted"; } yield* second.runtime.interruptTurn({ providerThread: second.thread, @@ -2262,7 +2267,7 @@ it.live.each(["accepted", "storage", "storage-accepted"] as const)( const final = (yield* other.read)[1]!; assert.equal(final.state, mode === "storage" ? "failed" : "interrupted"); assert.equal(final.submissionRejected === true, mode === "storage"); - assert.equal(remote.control.interruptPosts, mode === "storage" ? 0 : 1); + assert.equal(remote.control.interruptPosts, mode === "storage-accepted" ? 1 : 0); } else { assert.equal(saved.remoteState, "running"); } diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts index 1fda3337c8b8..8433b977ad5e 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -1513,7 +1513,13 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { "Cloud admission has no confirmed session ID. Remote Stop is unavailable; task and billing status remain unknown.", ); yield* watch; - if (active.interruptRequested) return true; + if (active.interruptRequested) { + // Another observer already persisted Stop. Resume observation + // without sending a duplicate interrupt. + admissionStoragePaused = false; + admissionFailures = 0; + return true; + } yield* save({ ...active, interruptRequested: true }); // A successful write retires the local storage pause so this // explicit Stop can observe remote confirmation. Keep durable From 1fb6d4d7eccabedb1ac145bb726e93b254e7cb06 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sun, 4 Oct 2026 15:43:06 +0000 Subject: [PATCH 22/44] fix(kilo): keep recovery and cleanup observable during failures --- .../src/components/LocalWorkspaceNotice.tsx | 2 +- .../Adapters/KiloCloudAdapterV2.test.ts | 336 +++++++++++++++++- .../Adapters/KiloCloudAdapterV2.ts | 140 ++++++-- .../src/provider/OpenCodeServerLedger.test.ts | 171 +++++---- .../src/provider/OpenCodeServerLedger.ts | 20 +- .../src/provider/kilo/KiloCloudJournal.ts | 14 +- 6 files changed, 573 insertions(+), 110 deletions(-) diff --git a/apps/mobile/src/components/LocalWorkspaceNotice.tsx b/apps/mobile/src/components/LocalWorkspaceNotice.tsx index 7bbb4d78a48d..2df9ce1458ee 100644 --- a/apps/mobile/src/components/LocalWorkspaceNotice.tsx +++ b/apps/mobile/src/components/LocalWorkspaceNotice.tsx @@ -17,7 +17,7 @@ export function LocalWorkspaceNotice({ {state === "loading" ? ( - + Loading conversation workspace... diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index 6672ea5f7a46..37d384047b7c 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -89,6 +89,7 @@ const fixture = Effect.acquireRelease( }); const control = { prepareStatus: 200, + rejectAcceptedPrepare: false, parkPrepare: false, parkedPrepare: undefined as NodeHttp.ServerResponse | undefined, prepareSeen: undefined as (() => void) | undefined, @@ -99,6 +100,7 @@ const fixture = Effect.acquireRelease( parkedList: undefined as NodeHttp.ServerResponse | undefined, listSeen: undefined as (() => void) | undefined, listClosed: undefined as (() => void) | undefined, + sandboxActive: undefined as boolean | undefined, resultReads: 0, completeAfterResultReads: 0, sessionStatus: 200, @@ -232,6 +234,11 @@ const fixture = Effect.acquireRelease( messages: [{ id: input.initialMessageId!, prompt: input.prompt! }], }; conversations.set(state.cloud, state); + if (control.rejectAcceptedPrepare) { + response.writeHead(400); + response.end(); + return; + } if (control.dropNextPrepare) { control.dropNextPrepare = false; response.destroy(); @@ -313,7 +320,7 @@ const fixture = Effect.acquireRelease( if (control.parkSend) { control.parkedSend = response; response.once("close", () => { - control.parkedSend = undefined; + if (control.parkedSend === response) control.parkedSend = undefined; }); response.on("error", () => {}); control.sendSeen?.(); @@ -363,14 +370,14 @@ const fixture = Effect.acquireRelease( }); if (operation === "cloudAgentNext.getSandboxStatus") return reply({ - status: control.status === "running" ? "active" : "sleeping", + status: (control.sandboxActive ?? control.status === "running") ? "active" : "sleeping", observedAt: 1, inactivityTimeoutMs: null, estimatedSleepAt: null, }); if (operation === "cloudAgentNext.getComputeBillingStatus") return reply({ - phase: control.status === "running" ? "active" : "idle", + phase: (control.sandboxActive ?? control.status === "running") ? "active" : "idle", attribution: "session", estimatedHourlyRateMicrodollars: 0, estimatedIntervalAmountMicrodollars: 0, @@ -1258,6 +1265,9 @@ it.live( const admissionHarness = Effect.fn("admissionHarness")(function* ( remote: Effect.Success, directory: string, + decorateJournal?: ( + journal: Effect.Success>, + ) => Effect.Success>, ) { const instanceId = ProviderInstanceId.make("cloud-admission"); const threadId = ThreadId.make("admission-thread"); @@ -1283,7 +1293,7 @@ const admissionHarness = Effect.fn("admissionHarness")(function* ( repository: "fixture/repo", branch: "main", client, - journal, + journal: decorateJournal ? decorateJournal(journal) : journal, }); const initial: import("@t3tools/contracts").OrchestrationV2ProviderThread = { id: ProviderThreadId.make("admission-provider-thread"), @@ -2219,6 +2229,17 @@ it.live.each(["accepted", "storage", "storage-accepted", "storage-interrupted"] remote.control.status = "running"; remote.control.parkSend = true; const second = yield* harness.open; + const monitored = yield* Deferred.make(); + yield* second.runtime.events.pipe( + Stream.runForEach((event) => + event.type === "provider_thread.updated" && + event.providerThread.nativeMetadata?.cloudExecution?.sandbox === "active" && + event.providerThread.nativeMetadata.cloudExecution.billing === "active" + ? Deferred.succeed(monitored, undefined) + : Effect.void, + ), + Effect.forkScoped, + ); const db = yield* Effect.acquireRelease( Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), (db) => Effect.sync(() => db.close()), @@ -2245,6 +2266,8 @@ it.live.each(["accepted", "storage", "storage-accepted", "storage-interrupted"] assert.deepEqual(saved.binding, originalBinding); yield* second.runtime.startTurn(harness.turn(second.thread, 3)).pipe(Effect.flip); if (mode !== "accepted") { + assert.isTrue(yield* second.runtime.hasPendingBackgroundWork!); + yield* Deferred.await(monitored); const stopped = yield* second.runtime .interruptTurn({ providerThread: second.thread, providerTurnId: saved.providerTurn.id }) .pipe(Effect.flip); @@ -2276,3 +2299,308 @@ it.live.each(["accepted", "storage", "storage-accepted", "storage-interrupted"] }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), 20_000, ); + +it.effect.each(["start", "stop"] as const)( + "does not report a failed rejection save after %s completes past its deadline", + (mode) => + Effect.gen(function* () { + const remote = yield* fixture; + remote.control.prepareStatus = 400; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const committed = yield* Deferred.make(); + const release = yield* Deferred.make(); + let park = mode === "start"; + const harness = yield* admissionHarness(remote, directory, (journal) => ({ + ...journal, + save: (intent) => + journal + .save(intent) + .pipe( + Effect.tap((saved) => + park && saved.submissionRejected + ? Deferred.succeed(committed, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ) + : Effect.void, + ), + ), + })); + const opened = yield* harness.open; + const db = yield* Effect.acquireRelease( + Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), + (db) => Effect.sync(() => db.close()), + ); + if (mode === "stop") { + db.exec( + "CREATE TRIGGER reject_outcome BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN SELECT RAISE(FAIL, 'fixture rejection write failure'); END", + ); + yield* opened.runtime.startTurn(harness.turn(opened.thread)); + db.exec("DROP TRIGGER reject_outcome"); + park = true; + } + const pending = yield* ( + mode === "start" + ? opened.runtime.startTurn(harness.turn(opened.thread)) + : opened.runtime.interruptTurn({ + providerThread: opened.thread, + providerTurnId: (yield* harness.journal.read)[0]!.providerTurn.id, + }) + ).pipe(Effect.forkScoped); + yield* Deferred.await(committed); + assert.equal((yield* harness.journal.read)[0]!.state, "failed"); + yield* TestClock.adjust("8 seconds"); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(pending); + // Emit a new thread boundary so every preceding queued status has been read. + park = false; + remote.control.preflightStatus = 503; + yield* opened.runtime.startTurn(harness.turn(opened.thread, 2)); + const boundary = (yield* harness.journal.read)[1]!.providerTurn.id; + const events = yield* opened.runtime.events.pipe( + Stream.takeUntil( + (event) => event.type === "provider_turn.updated" && event.providerTurn.id === boundary, + ), + Stream.runCollect, + ); + const terminal = events.filter((event) => event.type === "turn.terminal"); + assert.equal(terminal.length, 1); + const after = events.slice(events.indexOf(terminal[0]!) + 1); + assert.isFalse( + after.some( + (event) => + event.type === "provider_session.updated" && event.providerSession.lastError !== null, + ), + ); + assert.equal(remote.control.preparePosts, 1); + assert.equal(remote.control.interruptPosts, 0); + assert.isFalse(yield* opened.runtime.hasPendingBackgroundWork!); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live.each(["start", "stop"] as const)( + "retries a real first failed rejection write during %s", + (mode) => + Effect.gen(function* () { + const remote = yield* fixture; + remote.control.prepareStatus = 400; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const harness = yield* admissionHarness(remote, directory); + const opened = yield* harness.open; + const db = yield* Effect.acquireRelease( + Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), + (db) => Effect.sync(() => db.close()), + ); + if (mode === "stop") { + db.exec( + "CREATE TRIGGER reject_outcome BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN SELECT RAISE(FAIL, 'fixture rejection write failure'); END", + ); + yield* opened.runtime.startTurn(harness.turn(opened.thread)); + db.exec("DROP TRIGGER reject_outcome"); + } + db.exec( + "CREATE TABLE fail_once (armed INTEGER); INSERT INTO fail_once VALUES (1); CREATE TRIGGER reject_once BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 AND EXISTS(SELECT 1 FROM fail_once) BEGIN DELETE FROM fail_once; SELECT RAISE(IGNORE); END; CREATE TABLE rejection_attempts (attempt INTEGER); CREATE TRIGGER count_rejections BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN INSERT INTO rejection_attempts VALUES (1); END", + ); + if (mode === "start") yield* opened.runtime.startTurn(harness.turn(opened.thread)); + else + yield* opened.runtime.interruptTurn({ + providerThread: opened.thread, + providerTurnId: (yield* harness.journal.read)[0]!.providerTurn.id, + }); + assert.equal(db.prepare("SELECT COUNT(*) AS count FROM fail_once").get()!.count, 0); + assert.equal(db.prepare("SELECT COUNT(*) AS count FROM rejection_attempts").get()!.count, 2); + assert.equal((yield* harness.journal.read)[0]!.state, "failed"); + assert.isTrue((yield* harness.journal.read)[0]!.submissionRejected); + assert.equal(remote.control.preparePosts, 1); + assert.equal(remote.control.interruptPosts, 0); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.live.each(["resume", "read-failure"] as const)( + "handles concurrent %s after a recovery pause write loses CAS", + (mode) => + Effect.gen(function* () { + const { remote, directory } = yield* uncertainAdmission; + remote.control.listStatus = 503; + const other = yield* Journal.make(`${directory}/journal`); + yield* other.save({ ...(yield* other.read)[0]!, admissionRecoveryFailures: 2 }); + const db = yield* Effect.acquireRelease( + Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), + (db) => Effect.sync(() => db.close()), + ); + let hidden = false; + let raced = false; + const harness = yield* admissionHarness(remote, directory, (journal) => ({ + ...journal, + save: (intent) => + Effect.gen(function* () { + if (intent.admissionRecoveryPaused && !raced) { + raced = true; + yield* other.save({ + ...(yield* other.read)[0]!, + admissionRecoveryPaused: false, + admissionRecoveryFailures: 0, + }); + } + return yield* journal.save(intent).pipe( + Effect.tapError(() => + Effect.sync(() => { + if (mode === "read-failure" && !hidden) { + db.exec("ALTER TABLE intents RENAME TO temporarily_unavailable"); + hidden = true; + } + }), + ), + ); + }), + })); + const opened = yield* harness.open; + const observed = yield* opened.runtime + .readThreadSnapshot({ providerThread: opened.thread }) + .pipe(Effect.exit); + if (hidden) db.exec("ALTER TABLE temporarily_unavailable RENAME TO intents"); + assert.equal(Exit.isFailure(observed), mode === "read-failure"); + assert.isTrue(raced); + const saved = (yield* other.read)[0]!; + assert.isFalse(saved.admissionRecoveryPaused); + assert.equal(saved.admissionRecoveryFailures, 0); + assert.equal(yield* opened.runtime.hasPendingBackgroundWork!, mode === "resume"); + if (mode === "read-failure") { + const events = yield* opened.runtime.events.pipe( + Stream.filter( + (event) => + event.type === "provider_session.updated" && + event.providerSession.status === "waiting", + ), + Stream.take(1), + Stream.runCollect, + ); + const event = events[0]!; + assert.isTrue( + event.type === "provider_session.updated" && + event.providerSession.lastError?.includes("journal cannot be updated"), + ); + remote.control.listStatus = 200; + remote.control.hideAdmissions = false; + yield* opened.runtime.readThreadSnapshot({ providerThread: opened.thread }); + assert.equal((yield* other.read)[0]!.state, "completed"); + } + assert.equal(remote.control.preparePosts, 1); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); + +it.effect.each(["terminal-binding", "prepared", "prepared-write-failure"] as const)( + "Stop adopts newer %s after an unsaved rejection and keeps observing", + (mode) => + Effect.gen(function* () { + const remote = yield* fixture; + remote.control.rejectAcceptedPrepare = true; + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString( + `${directory}/data/kilo/auth.json`, + '{"kilo":{"type":"api","key":"synthetic"}}', + ); + const harness = yield* admissionHarness(remote, directory); + const opened = yield* harness.open; + const db = yield* Effect.acquireRelease( + Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), + (db) => Effect.sync(() => db.close()), + ); + db.exec( + "CREATE TRIGGER reject_outcome BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN SELECT RAISE(FAIL, 'fixture rejection write failure'); END", + ); + yield* opened.runtime.startTurn(harness.turn(opened.thread)); + const saved = (yield* harness.journal.read)[0]!; + const found = yield* harness.client.findAdmission("fixture/repo", saved.messageId); + assert.isNotNull(found); + const binding = yield* harness.client.bind(found!, "fixture/repo", saved.messageId, "main"); + const other = yield* Journal.make(`${directory}/journal`); + if (mode === "terminal-binding") { + remote.control.sandboxActive = true; + yield* other.save({ + ...saved, + prepared: found, + binding, + state: "completed", + remoteState: "completed", + providerTurn: { + ...saved.providerTurn, + status: "completed", + completedAt: yield* DateTime.now, + }, + }); + } else yield* other.save({ ...saved, prepared: found }); + const finished = yield* Deferred.make(); + const monitoring = yield* Deferred.make(); + const sleeping = yield* Deferred.make(); + let terminalEvents = 0; + yield* opened.runtime.events.pipe( + Stream.runForEach((event) => + Effect.gen(function* () { + if (event.type === "turn.terminal") { + terminalEvents++; + yield* Deferred.succeed(finished, undefined); + } + if (event.type === "provider_thread.updated") { + const cloud = event.providerThread.nativeMetadata?.cloudExecution; + if (cloud?.sandbox === "active" && cloud.billing === "active") + yield* Deferred.succeed(monitoring, undefined); + if (cloud?.sandbox === "sleeping" && cloud.billing === "idle") + yield* Deferred.succeed(sleeping, undefined); + } + }), + ), + Effect.forkScoped, + ); + const stop = opened.runtime.interruptTurn({ + providerThread: opened.thread, + providerTurnId: saved.providerTurn.id, + }); + if (mode === "prepared-write-failure") { + db.exec( + "CREATE TRIGGER reject_all BEFORE UPDATE ON intents BEGIN SELECT RAISE(FAIL, 'fixture write failure'); END", + ); + const failure = yield* stop.pipe(Effect.flip); + assert.include(failure.message, "journal cannot be updated"); + assert.isFalse(yield* opened.runtime.hasPendingBackgroundWork!); + assert.equal((yield* other.read)[0]!.state, "admission_unknown"); + db.exec("DROP TRIGGER reject_all"); + } + if (mode === "terminal-binding") yield* stop; + else { + const unavailable = yield* stop.pipe(Effect.flip); + assert.include(unavailable.message, "no confirmed session ID"); + } + yield* Deferred.await(finished); + if (mode === "terminal-binding") { + yield* Deferred.await(monitoring); + assert.isTrue(yield* opened.runtime.hasPendingBackgroundWork!); + remote.control.sandboxActive = false; + yield* TestClock.adjust("15 seconds"); + } + yield* Deferred.await(sleeping); + assert.isFalse(yield* opened.runtime.hasPendingBackgroundWork!); + assert.equal(terminalEvents, 1); + assert.equal(remote.control.preparePosts, 1); + assert.equal(remote.control.sendPosts, 0); + assert.equal(remote.control.interruptPosts, 0); + assert.equal((yield* other.read)[0]!.state, "completed"); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), + 20_000, +); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts index 8433b977ad5e..c33bb46da8e4 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -147,6 +147,9 @@ const wire = (effect: Effect.Effect) => const encode = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const isCloudError = Schema.is(KiloCloudError); const isRecord = Schema.is(Schema.Record(Schema.String, Schema.Unknown)); +const isJournalError = Schema.is(Journal.CloudJournalError); +const isJournalConflict = (cause: unknown) => + isRecord(cause) && isJournalError(cause.cause) && cause.cause.reason === "conflict"; export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { readonly instanceId: ProviderInstanceId; @@ -303,9 +306,15 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { const saved = { ...active, state: terminal, - providerTurn: { ...active.providerTurn, status: terminal, completedAt: at }, + providerTurn: { + ...active.providerTurn, + status: terminal, + completedAt: active.providerTurn.completedAt ?? at, + }, }; - yield* save(saved); + // A terminal record adopted from another observer is already durable. + if (active.state !== terminal || active.providerTurn.status !== terminal) + yield* save(saved); taskState = notSubmitted || rejected ? "not_started" : (saved.remoteState ?? terminal); yield* options.client.forgetAdmission(options.repository, saved.messageId); if (thread?.nativeMetadata?.cloudExecution) { @@ -686,6 +695,29 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { } return false; }); + // Retry local persistence only. Never repeat a paid customer request. + const settleKnownOutcome = Effect.gen(function* () { + yield* refreshIntent; + return yield* finishKnownOutcome; + }).pipe( + Effect.retry({ times: 1 }), + Effect.timeout("8 seconds"), + // The timeout can lose a race to an uninterruptible terminal commit. + Effect.catch((cause) => (!active ? Effect.succeed(true) : Effect.fail(cause))), + ); + const resumeLocalStorage = Effect.gen(function* () { + yield* refreshIntent; + if (yield* finishKnownOutcome) return true; + if (active) yield* save(active); + admissionStoragePaused = false; + admissionFailures = 0; + admissionProbeAt = 0; + return false; + }).pipe( + Effect.retry({ times: 1 }), + Effect.timeout("8 seconds"), + Effect.catch((cause) => (!active ? Effect.succeed(true) : Effect.fail(cause))), + ); const recoverAdmission = Effect.gen(function* () { yield* refreshIntent; if (!active || (yield* finishKnownOutcome)) return; @@ -733,6 +765,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ["recovery_incomplete", "recovery_limit", "wrong_owner"].includes(cause.reason)); // Count and pause locally before further I/O: an unavailable journal // must not let the outer reconciliation timeout erase all progress. + const storageWasPaused = admissionStoragePaused; if (paused) { admissionStoragePaused = true; monitorSandbox = false; @@ -747,6 +780,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { // Only an explicit resume can clear that newer durable pause. admissionFailures = Math.max(admissionFailures, active.admissionRecoveryFailures ?? 0); paused ||= active.admissionRecoveryPaused === true || admissionFailures >= 3; + let conflicted = false; yield* save({ ...active, admissionRecoveryFailures: admissionFailures, @@ -758,20 +792,38 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { admissionStoragePaused = false; }), ), - Effect.catch(() => - Effect.sync(() => { - if (paused) admissionStoragePaused = true; - }), + Effect.catch((writeCause) => + Effect.gen(function* () { + if (isJournalConflict(writeCause)) { + // This probe lost to another observer. Adopt its pause/resume + // instead of reapplying counters computed before its write. + yield* refreshIntent.pipe(Effect.timeout("500 millis")); + conflicted = true; + admissionFailures = active?.admissionRecoveryFailures ?? 0; + paused = active?.admissionRecoveryPaused === true; + admissionStoragePaused = storageWasPaused; + } else if (paused) admissionStoragePaused = true; + }).pipe( + Effect.catch(() => + Effect.sync(() => { + admissionStoragePaused = true; + paused = true; + conflicted = false; + }), + ), + ), ), ); if (paused) { monitorSandbox = false; yield* status( - admissionStoragePaused - ? "Cloud recovery is paused because its journal cannot be updated. Restore local storage and reopen history. Submission and billing remain unknown; no request was resubmitted." - : isCloudError(cause) && cause.reason === "recovery_limit" - ? "Cloud recovery reached the 100-page history limit. Reopening repeats this limit; contact support to resolve the existing operation. Submission and billing remain unknown; do not submit again." - : `Cloud admission recovery is incomplete and automatic scanning is paused${isCloudError(cause) ? ` (${cause.recoveryCause ?? cause.reason})` : ""}. Reopen history to retry reads. Submission and billing remain unknown; do not submit again.`, + conflicted + ? "Cloud recovery changed in another observer. Reopen history to retry reads; no request was resubmitted." + : admissionStoragePaused + ? "Cloud recovery is paused because its journal cannot be updated. Restore local storage and reopen history. Submission and billing remain unknown; no request was resubmitted." + : isCloudError(cause) && cause.reason === "recovery_limit" + ? "Cloud recovery reached the 100-page history limit. Reopening repeats this limit; contact support to resolve the existing operation. Submission and billing remain unknown; do not submit again." + : `Cloud admission recovery is incomplete and automatic scanning is paused${isCloudError(cause) ? ` (${cause.recoveryCause ?? cause.reason})` : ""}. Reopen history to retry reads. Submission and billing remain unknown; do not submit again.`, ); } }), @@ -1073,7 +1125,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { while (hasBackgroundWork()) { const pollStartedAt = yield* Clock.currentTimeMillis; yield* watchEvents; - if (active || needsHistoryRestore) + if (!admissionStoragePaused && (active || needsHistoryRestore)) yield* gate .withPermit(reconcile().pipe(Effect.timeout("10 seconds"))) .pipe( @@ -1450,18 +1502,16 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { // Commit the rejection and terminal state together. Refresh on // a CAS conflict, but never retry the paid request. If storage // stays unavailable, retain the outcome only in this adapter. - yield* Effect.gen(function* () { - yield* refreshIntent; - yield* finishKnownOutcome; - }).pipe( - Effect.retry({ times: 1 }), - Effect.timeout("8 seconds"), - Effect.catch(() => + yield* settleKnownOutcome.pipe( + Effect.catch((cause) => Effect.gen(function* () { + if (!active) return; admissionStoragePaused = true; - monitorSandbox = false; + if (!binding) monitorSandbox = false; yield* status( - "Kilo Cloud rejected this request, but the outcome could not be saved. Restore local storage and reopen history. The reservation remains held; no request was resubmitted.", + isJournalConflict(cause) + ? "Cloud rejection recovery changed concurrently. Reopen history to retry local recovery; no request was resubmitted." + : "Kilo Cloud rejected this request, but the outcome could not be saved. Restore local storage and reopen history. The reservation remains held; no request was resubmitted.", ); }), ), @@ -1482,19 +1532,42 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { Effect.gen(function* () { yield* owned(request.providerThread); if (!active || active.providerTurn.id !== request.providerTurnId) return false; - if (rejectedOperationKey === active.operationKey || active.submissionRejected) { - const finished = yield* Effect.gen(function* () { - yield* refreshIntent; - return yield* finishKnownOutcome; - }).pipe( - Effect.timeout("8 seconds"), - Effect.mapError(() => + if ( + rejectedOperationKey === active.operationKey || + active.submissionRejected || + admissionStoragePaused + ) { + const finished = yield* settleKnownOutcome.pipe( + Effect.mapError((cause) => error( - "Kilo Cloud rejected this request, but its outcome cannot be saved. Restore local storage and reopen history. No remote interrupt was sent.", + isJournalConflict(cause) + ? "Cloud rejection recovery changed concurrently. Reopen history to retry local recovery. No remote interrupt was sent." + : "Kilo Cloud rejected this request, but its outcome cannot be saved. Restore local storage and reopen history. No remote interrupt was sent.", ), ), ); - if (finished || !active) return false; + if (finished || !active) { + yield* watch; + return false; + } + if (admissionStoragePaused) { + const settled = yield* resumeLocalStorage.pipe( + Effect.mapError((cause) => + error( + isJournalConflict(cause) + ? "Cloud recovery changed concurrently. Reopen history to retry. No remote interrupt was sent." + : "Cloud recovery remains paused because its journal cannot be updated. Restore local storage and reopen history. No remote interrupt was sent.", + ), + ), + ); + if (settled || !active) { + yield* watch; + return false; + } + } + // Newer prepared admission is authoritative, but not a binding. + // Resume its read-only recovery; Stop can be retried after binding. + if (!binding) yield* watch; } if (active.submissionPhase === "preflight") { yield* finish("interrupted"); @@ -1587,10 +1660,13 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { admissionFailures = 0; admissionStoragePaused = false; }).pipe( + Effect.retry({ times: 1 }), Effect.timeout("8 seconds"), - Effect.mapError(() => + Effect.mapError((cause) => error( - "Cloud recovery remains paused because its journal cannot be updated. Restore local storage and reopen history. No request was resubmitted.", + isJournalConflict(cause) + ? "Cloud recovery changed concurrently. Reopen history to retry reads. No request was resubmitted." + : "Cloud recovery remains paused because its journal cannot be updated. Restore local storage and reopen history. No request was resubmitted.", ), ), ); diff --git a/apps/server/src/provider/OpenCodeServerLedger.test.ts b/apps/server/src/provider/OpenCodeServerLedger.test.ts index c2beb3ca26e0..77e091d753ba 100644 --- a/apps/server/src/provider/OpenCodeServerLedger.test.ts +++ b/apps/server/src/provider/OpenCodeServerLedger.test.ts @@ -243,69 +243,118 @@ describe.skipIf(observedPlatforms.length === 0)("OpenCode server startup", () => ); }); -it.live.skipIf(hostPlatform !== "linux")( - "serializes boot and profile reapers until a TERM-resistant group has stopped", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ledger-overlap-" }); - const group = yield* Effect.acquireRelease( - Effect.gen(function* () { - const ready = yield* Deferred.make(); - const exited = yield* Deferred.make(); - const outputClosed = yield* Deferred.make(); - const leader = NodeChildProcess.spawn( - process.execPath, - [ - "-e", - ` +for (const cancel of [false, true]) + it.live.skipIf(hostPlatform !== "linux")( + `serializes boot and profile reapers through group cleanup with cancellation=${cancel}`, + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ledger-overlap-" }); + const group = yield* Effect.acquireRelease( + Effect.gen(function* () { + const ready = yield* Deferred.make(); + const exited = yield* Deferred.make(); + const outputClosed = yield* Deferred.make(); + const leader = NodeChildProcess.spawn( + process.execPath, + [ + "-e", + ` const {spawn}=require('node:child_process'); const child=spawn(process.execPath,['-e',"process.on('SIGTERM',()=>{});process.send(process.pid);setInterval(()=>{},1000)"],{stdio:['ignore',process.stdout,'ignore','ipc']}); child.once('message',pid=>process.send(pid)); setInterval(()=>{},1000); `, - ], - { detached: true, stdio: ["ignore", "pipe", "ignore", "ipc"] }, - ); - leader.once("message", (pid) => Deferred.doneUnsafe(ready, Effect.succeed(Number(pid)))); - leader.once("exit", () => Deferred.doneUnsafe(exited, Effect.void)); - leader.stdout!.once("end", () => Deferred.doneUnsafe(outputClosed, Effect.void)); - leader.stdout!.resume(); - return { - pid: leader.pid!, - ready: Deferred.await(ready), - exited: Deferred.await(exited), - outputClosed: Deferred.await(outputClosed), - }; - }), - (group) => killGroup(group.pid), - ); - const member = yield* group.ready; - yield* recordFromDeadServer(stateDir, group); - const boot = yield* OpenCodeServerLedger.make({ stateDir }); - const profile = yield* OpenCodeServerLedger.make({ stateDir }); - const complete = yield* Deferred.make(); - const first = yield* boot.reapOrphans.pipe( - Effect.andThen(Deferred.succeed(complete, undefined)), - Effect.forkScoped, - ); - yield* group.exited; // TERM killed the recorded leader; its child ignores TERM. - const running = fs.readFileString(`/proc/${member}/stat`).pipe( - Effect.map((stat) => !stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")), - Effect.orElseSucceed(() => false), - ); - expect(yield* running).toBe(true); - yield* profile.reapOrphans; - expect(yield* Deferred.isDone(complete)).toBe(true); - expect(yield* running).toBe(false); - yield* group.outputClosed; - yield* Fiber.join(first); - // Coordination is released after the scan, not cached forever. - const later = yield* spawnGroup(SERVE_ARGS); - yield* recordFromDeadServer(stateDir, later); - yield* profile.reapOrphans; - yield* later.exited; - expect(groupExists(later.pid)).toBe(false); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - 20_000, -); + ], + { detached: true, stdio: ["ignore", "pipe", "ignore", "ipc"] }, + ); + leader.once("message", (pid) => + Deferred.doneUnsafe(ready, Effect.succeed(Number(pid))), + ); + leader.once("exit", () => Deferred.doneUnsafe(exited, Effect.void)); + leader.stdout!.once("end", () => Deferred.doneUnsafe(outputClosed, Effect.void)); + leader.stdout!.resume(); + return { + pid: leader.pid!, + ready: Deferred.await(ready), + exited: Deferred.await(exited), + outputClosed: Deferred.await(outputClosed), + }; + }), + (group) => killGroup(group.pid), + ); + const member = yield* group.ready; + yield* recordFromDeadServer(stateDir, group); + const boot = yield* OpenCodeServerLedger.make({ stateDir }); + const profile = yield* OpenCodeServerLedger.make({ stateDir }); + const complete = yield* Deferred.make(); + const first = yield* boot.reapOrphans.pipe( + Effect.ensuring(Deferred.succeed(complete, undefined)), + Effect.forkScoped, + ); + yield* group.exited; // TERM killed the recorded leader; its child ignores TERM. + const running = fs.readFileString(`/proc/${member}/stat`).pipe( + Effect.map((stat) => !stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")), + Effect.orElseSucceed(() => false), + ); + expect(yield* running).toBe(true); + if (cancel) yield* Fiber.interrupt(first).pipe(Effect.forkScoped); + yield* profile.reapOrphans; + expect(yield* Deferred.isDone(complete)).toBe(true); + expect(yield* running).toBe(false); + yield* group.outputClosed; + const outcome = yield* Fiber.await(first); + expect(Exit.isFailure(outcome)).toBe(cancel); + // Coordination is released after the scan, not cached forever. + const later = yield* spawnGroup(SERVE_ARGS); + yield* recordFromDeadServer(stateDir, later); + yield* profile.reapOrphans; + yield* later.exited; + expect(groupExists(later.pid)).toBe(false); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + 20_000, + ); + +describe.skipIf(hostPlatform !== "linux")("interruptible discovery", () => { + it.live.each(["readDirectory", "readFileString", "remove"] as const)( + "cancels stalled ledger %s and releases the handoff gate", + (operation) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const stateDir = yield* fs.makeTempDirectoryScoped(); + const group = yield* spawnGroup(SERVE_ARGS); + yield* recordFromDeadServer(stateDir, group); + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const stall = (kind: typeof operation, path: string) => + kind === operation && path.includes("opencode-servers") + ? Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))) + : Effect.void; + const stalled: FileSystem.FileSystem = { + ...fs, + readDirectory: (...args) => + stall("readDirectory", args[0]).pipe(Effect.andThen(fs.readDirectory(...args))), + readFileString: (...args) => + stall("readFileString", args[0]).pipe(Effect.andThen(fs.readFileString(...args))), + remove: (...args) => stall("remove", args[0]).pipe(Effect.andThen(fs.remove(...args))), + }; + const blocked = yield* OpenCodeServerLedger.make({ stateDir }).pipe( + Effect.provideService(FileSystem.FileSystem, stalled), + ); + const second = yield* OpenCodeServerLedger.make({ stateDir }); + const running = yield* blocked.reapOrphans.pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const cancelled = yield* Fiber.interrupt(running).pipe( + Effect.timeout("1 second"), + Effect.exit, + Effect.ensuring(Deferred.succeed(release, undefined)), + ); + yield* Fiber.await(running); + expect(Exit.isSuccess(cancelled)).toBe(true); + yield* second.reapOrphans; + yield* group.exited; + expect(groupExists(group.pid)).toBe(false); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + 20_000, + ); +}); diff --git a/apps/server/src/provider/OpenCodeServerLedger.ts b/apps/server/src/provider/OpenCodeServerLedger.ts index 702506702a5f..7f663eb3551e 100644 --- a/apps/server/src/provider/OpenCodeServerLedger.ts +++ b/apps/server/src/provider/OpenCodeServerLedger.ts @@ -286,12 +286,14 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { pid: entry.pgid, port: entry.port, }); - signalGroup(entry.pgid, "SIGTERM"); - for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS && groupExists(entry.pgid); attempt++) { - yield* Effect.sleep(STOP_POLL_INTERVAL); - } - // The group never emptied, so its pgid cannot have been reused. - if (groupExists(entry.pgid)) signalGroup(entry.pgid, "SIGKILL"); + yield* Effect.gen(function* () { + signalGroup(entry.pgid, "SIGTERM"); + for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS && groupExists(entry.pgid); attempt++) { + yield* Effect.sleep(STOP_POLL_INTERVAL); + } + // The group never emptied, so its pgid cannot have been reused. + if (groupExists(entry.pgid)) signalGroup(entry.pgid, "SIGKILL"); + }).pipe(Effect.uninterruptible); }); const reapEntry = (entryPath: string) => @@ -331,9 +333,9 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { entry.users++; return entry; }), - // Once we signal a verified group, cancellation must not release the gate - // before its bounded TERM/KILL cleanup finishes. Waiting callers can cancel. - (entry) => entry.gate.withPermit(reapOnce.pipe(Effect.uninterruptible)), + // Discovery and filesystem cleanup remain interruptible. stopOrphan holds + // this permit through its bounded post-signal process-group cleanup. + (entry) => entry.gate.withPermit(reapOnce), (entry) => Effect.sync(() => { if (--entry.users === 0) reapers.delete(key); diff --git a/apps/server/src/provider/kilo/KiloCloudJournal.ts b/apps/server/src/provider/kilo/KiloCloudJournal.ts index cba2bd9d0d32..90399fabd79a 100644 --- a/apps/server/src/provider/kilo/KiloCloudJournal.ts +++ b/apps/server/src/provider/kilo/KiloCloudJournal.ts @@ -71,9 +71,11 @@ export class CloudJournalError extends Schema.TaggedError()( "CloudJournalError", { operation: Schema.Literals(["read", "write"]), + reason: Schema.optional(Schema.Literal("conflict")), cause: Schema.optional(Schema.Defect()), }, ) {} +const isJournalError = Schema.is(CloudJournalError); const codec = Schema.fromJsonString(Schema.toCodecJson(CloudIntent)); const encode = Schema.encodeEffect(codec); const decode = Schema.decodeUnknownEffect(codec); @@ -132,7 +134,6 @@ export const make = Effect.fn("KiloCloudJournal.make")(function* (directory: str if (!row) return yield* new CloudJournalError({ operation: "write" }); const prior = yield* decode(row.body); if ( - prior.revision !== intent.revision || prior.accountId !== intent.accountId || prior.repository !== intent.repository || prior.branch !== intent.branch || @@ -146,12 +147,19 @@ export const make = Effect.fn("KiloCloudJournal.make")(function* (directory: str prior.state !== intent.state) ) return yield* new CloudJournalError({ operation: "write" }); + if (prior.revision !== intent.revision) + return yield* new CloudJournalError({ operation: "write", reason: "conflict" }); const next = { ...intent, revision: intent.revision + 1 }; const body = yield* encode(next); const updated = yield* sql`UPDATE intents SET state = ${intent.state}, body = ${body} WHERE operation_key = ${intent.operationKey} AND body = ${row.body} RETURNING operation_key`; - if (updated.length !== 1) return yield* new CloudJournalError({ operation: "write" }); + if (updated.length !== 1) + return yield* new CloudJournalError({ operation: "write", reason: "conflict" }); return next; - }).pipe(Effect.mapError((cause) => new CloudJournalError({ operation: "write", cause }))), + }).pipe( + Effect.mapError((cause) => + isJournalError(cause) ? cause : new CloudJournalError({ operation: "write", cause }), + ), + ), }; }); From 2aeba5ad59241c8f2c095bcccaad8e576c4d3c57 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sun, 4 Oct 2026 15:52:59 +0000 Subject: [PATCH 23/44] test(kilo): release delayed saves when assertions fail --- .../src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index 37d384047b7c..c83a1cbb26bb 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -2352,6 +2352,8 @@ it.effect.each(["start", "stop"] as const)( providerTurnId: (yield* harness.journal.read)[0]!.providerTurn.id, }) ).pipe(Effect.forkScoped); + // Release before the fork's interrupt finalizer if an assertion fails. + yield* Effect.addFinalizer(() => Deferred.succeed(release, undefined)); yield* Deferred.await(committed); assert.equal((yield* harness.journal.read)[0]!.state, "failed"); yield* TestClock.adjust("8 seconds"); From 1efb73b6e2f011e7764b025670c2f51c1f2bf61c Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sun, 4 Oct 2026 17:36:55 +0000 Subject: [PATCH 24/44] fix(kilo): require an explicit repository for live evidence --- apps/desktop/scripts/kilo-ui-evidence.mjs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/desktop/scripts/kilo-ui-evidence.mjs b/apps/desktop/scripts/kilo-ui-evidence.mjs index ca626241b9ce..94e82c7a9a09 100644 --- a/apps/desktop/scripts/kilo-ui-evidence.mjs +++ b/apps/desktop/scripts/kilo-ui-evidence.mjs @@ -10,6 +10,11 @@ import * as NodeUtil from "node:util"; import { chromium } from "playwright-core"; if (!process.env.KILO_BIN) throw new Error("KILO_BIN must point to the pinned local CLI"); +if (process.env.KILO_CLOUD_TEST_PROFILE && !process.env.KILO_CLOUD_TEST_REPOSITORY) { + throw new Error( + "KILO_CLOUD_TEST_REPOSITORY is required for an explicitly authorized live capture", + ); +} const root = NodePath.resolve(import.meta.dirname, "../../.."); const temporary = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-kilo-ui-")); const evidence = process.env.KILO_EVIDENCE_DIR ?? NodePath.join(temporary, "evidence"); @@ -122,7 +127,7 @@ await NodeFSP.writeFile( config: { enabled: !!process.env.KILO_CLOUD_TEST_PROFILE, profileDirectory: process.env.KILO_CLOUD_TEST_PROFILE ?? "", - repository: "thomasbrugman/t3-kilo-cloud-test", + repository: process.env.KILO_CLOUD_TEST_REPOSITORY ?? "synthetic/cloud-demo", branch: "main", model: "deepseek/deepseek-v4.1-flash", cloudConsent: !!process.env.KILO_CLOUD_TEST_PROFILE, From 8964a3546c392166a0977ffcddb98fe20245e459 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sun, 4 Oct 2026 17:42:20 +0000 Subject: [PATCH 25/44] fix(web): hide local project settings in cloud threads --- apps/web/src/components/ChatView.tsx | 10 ++++++++-- apps/web/src/components/chat/ThreadDetailsPanel.tsx | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 72b3d7c05dab..ab7bd660e39b 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -4996,6 +4996,11 @@ export default function ChatView(props: ChatViewProps) { keybinding?: string | null; keybindingCommand: KeybindingCommand | null; }): Promise> => { + if (isCloudThread) { + return AsyncResult.failure( + Cause.fail(new Error("Local project scripts are unavailable for cloud threads.")), + ); + } const updateResult = mapAtomCommandResult( await updateProjectScriptSettings({ environmentId, @@ -5082,6 +5087,7 @@ export default function ChatView(props: ChatViewProps) { : updateResult; }, [ + isCloudThread, allProjects, environmentById, environmentId, @@ -10770,13 +10776,13 @@ export default function ChatView(props: ChatViewProps) { threadId: activeThread.id, ...(draftId ? { draftId } : {}), activeProjectName: activeProject?.title, - activeProjectScripts: activeProject ? activeProjectScripts : undefined, + activeProjectScripts: !isCloudThread && activeProject ? activeProjectScripts : undefined, preferredScriptId: activeProject ? (lastInvokedScriptByProjectId[activeProject.id] ?? null) : null, keybindings, availableEditors, - showOpenInPicker, + showOpenInPicker: !isCloudThread && showOpenInPicker, gitCwd, isGitRepo, envLocked, diff --git a/apps/web/src/components/chat/ThreadDetailsPanel.tsx b/apps/web/src/components/chat/ThreadDetailsPanel.tsx index 5b79f0c00f57..5a3c0dcfba4b 100644 --- a/apps/web/src/components/chat/ThreadDetailsPanel.tsx +++ b/apps/web/src/components/chat/ThreadDetailsPanel.tsx @@ -164,7 +164,7 @@ export function ThreadDetailsPanel(props: ThreadDetailsPanelProps) { /> ) : null} - {density === "full" ? ( + {density === "full" && props.gitCwd !== null ? ( ) : null} From c36c134fe3c3aa12ed02a0f8217f2c009d95dc46 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Sun, 4 Oct 2026 20:30:16 +0000 Subject: [PATCH 26/44] fix(kilo): verify observed process exit before replacement --- .../Adapters/KiloAdapterV2.ts | 3 +- .../provider/kilo/KiloProcessCleanup.test.ts | 391 ++++++++++++++++++ .../src/provider/kilo/KiloProcessCleanup.ts | 185 +++++++++ .../provider/kilo/KiloRuntime.cleanup.test.ts | 241 +++++++++++ .../provider/kilo/KiloRuntime.live.test.ts | 44 +- apps/server/src/provider/kilo/KiloRuntime.ts | 92 +++-- docs/user/providers-kilo.md | 20 + 7 files changed, 942 insertions(+), 34 deletions(-) create mode 100644 apps/server/src/provider/kilo/KiloProcessCleanup.test.ts create mode 100644 apps/server/src/provider/kilo/KiloProcessCleanup.ts create mode 100644 apps/server/src/provider/kilo/KiloRuntime.cleanup.test.ts diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts index 19e2f6e58bb1..b812f79fcbc3 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts @@ -428,7 +428,8 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { yield* connection.exitCode.pipe( Effect.andThen( Effect.gen(function* () { - // The Stop owner publishes terminality only after the entire owned process group is gone. + // The Stop owner waits for runtime cleanup before publishing terminality. + // Linux verifies observed group members; this is not descendant containment. if (!active || active.interrupting) return; yield* connection.cleanup; yield* finish("failed", "Kilo process exited."); diff --git a/apps/server/src/provider/kilo/KiloProcessCleanup.test.ts b/apps/server/src/provider/kilo/KiloProcessCleanup.test.ts new file mode 100644 index 000000000000..fc9c107a0552 --- /dev/null +++ b/apps/server/src/provider/kilo/KiloProcessCleanup.test.ts @@ -0,0 +1,391 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as PlatformError from "effect/PlatformError"; +import * as TestClock from "effect/testing/TestClock"; +import * as Cleanup from "./KiloProcessCleanup.ts"; + +const stat = (state = "S", start = "123") => { + const fields = Array(20).fill("0"); + fields[0] = state; + fields[2] = "4242"; + fields[19] = start; + return `4242 (fixture child) ${fields.join(" ")}`; +}; +const records = (fs: FileSystem.FileSystem, root: string) => + Effect.gen(function* () { + const directories = yield* fs.readDirectory(`${root}/kilo-cleanup`); + return (yield* Effect.forEach(directories, (name) => + fs.readDirectory(`${root}/kilo-cleanup/${name}`), + )).flat(); + }); +const denied = () => + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "readFileString", + pathOrDescriptor: "/proc/4242/stat", + }); +const missing = () => + PlatformError.systemError({ + _tag: "NotFound", + module: "FileSystem", + method: "readFileString", + pathOrDescriptor: "/proc/4242/stat", + }); + +it.effect.each(["zombie", "gone", "reused", "permission", "malformed"] as const)( + "verifies %s without treating observation failure as absence", + (outcome) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + let stopped = false; + let repaired = false; + let starts = 0; + const fake: FileSystem.FileSystem = { + ...fs, + readDirectory: (path) => + path === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(path), + readFileString: (path, ...args) => + path !== "/proc/4242/stat" + ? fs.readFileString(path, ...args) + : Effect.suspend(() => { + if (!stopped) return Effect.succeed(stat()); + if (repaired || outcome === "zombie") return Effect.succeed(stat("Z")); + if (outcome === "gone") return Effect.fail(missing()); + if (outcome === "reused") return Effect.succeed(stat("S", "456")); + if (outcome === "permission") return Effect.fail(denied()); + return Effect.succeed("invalid stat"); + }), + }; + yield* Effect.gen(function* () { + const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); + const result = yield* Effect.exit( + cleanup.verify( + 4242, + Effect.sync(() => { + stopped = true; + }), + ), + ); + const bad = outcome === "permission" || outcome === "malformed"; + assert.equal(Exit.isFailure(result), bad); + // A new helper reads the real on-disk record, not the first helper's closure. + const reopened = yield* Cleanup.make({ profile: root, stateDir: root }); + const start = reopened.withStart(Effect.sync(() => ++starts)); + const admission = yield* Effect.exit(start); + assert.equal(Exit.isFailure(admission), bad); + assert.equal(starts, bad ? 0 : 1); + if (bad) { + assert.deepEqual(yield* records(fs, root), ["4242.json"]); + repaired = true; + yield* start; + assert.equal(starts, 1); + } + assert.deepEqual(yield* records(fs, root), []); + }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("timeout retains the recorded identities and recovery requires observed exit", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const stopping = yield* Deferred.make(); + let state = "S"; + const fake: FileSystem.FileSystem = { + ...fs, + readDirectory: (p) => (p === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(p)), + readFileString: (p, ...a) => + p === "/proc/4242/stat" ? Effect.sync(() => stat(state)) : fs.readFileString(p, ...a), + }; + yield* Effect.gen(function* () { + const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); + const fiber = yield* cleanup + .verify(4242, Deferred.succeed(stopping, undefined).pipe(Effect.asVoid)) + .pipe(Effect.forkScoped); + yield* Deferred.await(stopping); + yield* TestClock.adjust("3 seconds"); + assert.isTrue(Exit.isFailure(yield* Fiber.await(fiber))); + assert.deepEqual(yield* records(fs, root), ["4242.json"]); + state = "Z"; + let starts = 0; + yield* cleanup.withStart(Effect.sync(() => starts++)); + assert.equal(starts, 1); + }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("a failed reservation write cannot be bypassed by a successful directory read", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + let writesFail = true; + let state = "S"; + let signals = 0; + let starts = 0; + const fake: FileSystem.FileSystem = { + ...fs, + readDirectory: (p) => (p === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(p)), + readFileString: (p, ...args) => + p === "/proc/4242/stat" ? Effect.sync(() => stat(state)) : fs.readFileString(p, ...args), + writeFileString: (p, ...args) => + writesFail && p.endsWith("4242.json.tmp") + ? Effect.fail(denied()) + : fs.writeFileString(p, ...args), + }; + yield* Effect.gen(function* () { + const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); + yield* cleanup + .verify( + 4242, + Effect.sync(() => signals++), + ) + .pipe(Effect.flip); + writesFail = false; + const replacement = yield* Cleanup.make({ profile: root, stateDir: root }); + yield* replacement.withStart(Effect.sync(() => starts++)).pipe(Effect.flip); + assert.equal(signals, 0); + assert.equal(starts, 0); + // An unrelated profile is not blocked by this profile's storage failure. + const other = yield* Cleanup.make({ + profile: `${root}/other`, + stateDir: root, + }); + yield* other.withStart(Effect.sync(() => starts++)); + assert.equal(starts, 1); + state = "Z"; + yield* replacement.withStart(Effect.sync(() => starts++)); + assert.equal(starts, 2); + }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("incomplete snapshots recover only after a complete quiescent-group observation", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + let unreadable = true; + let state = "S"; + let starts = 0; + const fake: FileSystem.FileSystem = { + ...fs, + readDirectory: (p) => (p === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(p)), + readFileString: (p, ...args) => + p === "/proc/4242/stat" + ? Effect.suspend(() => (unreadable ? Effect.fail(denied()) : Effect.succeed(stat(state)))) + : fs.readFileString(p, ...args), + }; + yield* Effect.gen(function* () { + const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); + yield* cleanup + .verify(4242, Effect.die("must not signal without observation")) + .pipe(Effect.flip); + const reopened = yield* Cleanup.make({ profile: root, stateDir: root }); + const start = reopened.withStart(Effect.sync(() => starts++)); + yield* start.pipe(Effect.flip); + unreadable = false; + yield* start.pipe(Effect.flip); + assert.equal(starts, 0); + state = "Z"; + yield* start; + assert.equal(starts, 1); + assert.deepEqual(yield* records(fs, root), []); + }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect( + "first-open profile aliases share pending cleanup and preserve other account admission", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${root}/real`); + yield* fs.symlink(`${root}/real`, `${root}/alias`); + let unreadable = false; + let state = "S"; + const fake: FileSystem.FileSystem = { + ...fs, + readDirectory: (p) => (p === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(p)), + readFileString: (p, ...args) => + p === "/proc/4242/stat" + ? Effect.suspend(() => + unreadable ? Effect.fail(denied()) : Effect.succeed(stat(state)), + ) + : fs.readFileString(p, ...args), + }; + yield* Effect.gen(function* () { + const cleanup = yield* Cleanup.make({ + profile: `${root}/alias/new-profile`, + stateDir: root, + }); + yield* cleanup + .verify( + 4242, + Effect.sync(() => { + unreadable = true; + }), + ) + .pipe(Effect.flip); + const reopened = yield* Cleanup.make({ + profile: `${root}/real/new-profile`, + stateDir: root, + }); + let starts = 0; + yield* reopened.withStart(Effect.sync(() => starts++)).pipe(Effect.flip); + assert.equal(starts, 0); + // Even a malformed pending record in one account must not block another. + const [key] = yield* fs.readDirectory(`${root}/kilo-cleanup`); + const file = `${root}/kilo-cleanup/${key}/4242.json`; + const saved = yield* fs.readFileString(file); + yield* fs.writeFileString(file, "incomplete write"); + const other = yield* Cleanup.make({ profile: `${root}/other`, stateDir: root }); + yield* other.withStart(Effect.sync(() => starts++)); + assert.equal(starts, 1); + yield* fs.writeFileString(file, saved); + unreadable = false; + state = "Z"; + yield* reopened.withStart(Effect.sync(() => starts++)); + assert.equal(starts, 2); + }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("cancellation releases the lock but preserves uncertainty until observed exit", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const entered = yield* Deferred.make(); + let park = true; + let state = "S"; + const fake: FileSystem.FileSystem = { + ...fs, + readDirectory: (p) => + p === "/proc" + ? Effect.suspend(() => + park + ? Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)) + : Effect.succeed(["4242"]), + ) + : fs.readDirectory(p), + readFileString: (p, ...args) => + p === "/proc/4242/stat" ? Effect.sync(() => stat(state)) : fs.readFileString(p, ...args), + }; + yield* Effect.gen(function* () { + const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); + const fiber = yield* cleanup.verify(4242, Effect.void).pipe(Effect.forkScoped); + yield* Deferred.await(entered); + yield* Fiber.interrupt(fiber); + park = false; + let starts = 0; + const reopened = yield* Cleanup.make({ profile: root, stateDir: root }); + yield* reopened.withStart(Effect.sync(() => starts++)).pipe(Effect.flip); + assert.equal(starts, 0); + state = "Z"; + yield* reopened.withStart(Effect.sync(() => starts++)); + assert.equal(starts, 1); + }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect( + "holds the shared gate through observation and exit, without serializing session lifetime", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const scanning = yield* Deferred.make(); + const release = yield* Deferred.make(); + const requested = yield* Deferred.make(); + const observedAlive = yield* Deferred.make(); + let signalled = false; + const events: Array = []; + let state = "S"; + const fake: FileSystem.FileSystem = { + ...fs, + readDirectory: (p) => + p === "/proc" + ? Deferred.succeed(scanning, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as(["4242"]), + ) + : fs.readDirectory(p), + readFileString: (p, ...args) => + p === "/proc/4242/stat" + ? Effect.sync(() => { + const value = stat(state); + if (state === "Z") events.push("confirmed"); + else if (signalled) Deferred.doneUnsafe(observedAlive, Effect.void); + return value; + }) + : fs.readFileString(p, ...args), + }; + yield* Effect.gen(function* () { + const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); + const other = yield* Cleanup.make({ profile: root, stateDir: root }); + const stopping = yield* cleanup + .verify( + 4242, + Effect.sync(() => { + signalled = true; + }), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(scanning); + const replacement = yield* Deferred.succeed(requested, undefined).pipe( + Effect.andThen(other.withStart(Effect.sync(() => events.push("spawn")))), + Effect.forkScoped, + ); + yield* Deferred.await(requested); + yield* Deferred.succeed(release, undefined); + yield* Deferred.await(observedAlive); + assert.deepEqual(events, []); + state = "Z"; + yield* TestClock.adjust("10 millis"); + yield* Fiber.join(stopping); + yield* Fiber.join(replacement); + assert.deepEqual(events, ["confirmed", "spawn"]); + yield* other.withStart(Effect.sync(() => events.push("parallel-session"))); + assert.equal(events.at(-1), "parallel-session"); + }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + +it.effect("a partial snapshot write preserves a readable uncertainty record for recovery", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + let writes = 0; + let state = "S"; + const fake: FileSystem.FileSystem = { + ...fs, + readDirectory: (p) => (p === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(p)), + readFileString: (p, ...args) => + p === "/proc/4242/stat" ? Effect.sync(() => stat(state)) : fs.readFileString(p, ...args), + writeFileString: (p, ...args) => + Effect.suspend(() => { + if (p.endsWith(".tmp") && ++writes === 2) + return fs.writeFileString(p, "{partial").pipe(Effect.andThen(Effect.fail(denied()))); + return fs.writeFileString(p, ...args); + }), + }; + yield* Effect.gen(function* () { + const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); + yield* cleanup.verify(4242, Effect.die("no signal after partial write")).pipe(Effect.flip); + const reopened = yield* Cleanup.make({ profile: root, stateDir: root }); + let starts = 0; + const start = reopened.withStart(Effect.sync(() => starts++)); + yield* start.pipe(Effect.flip); + assert.equal(starts, 0); + state = "Z"; + yield* start; + assert.equal(starts, 1); + }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); diff --git a/apps/server/src/provider/kilo/KiloProcessCleanup.ts b/apps/server/src/provider/kilo/KiloProcessCleanup.ts new file mode 100644 index 000000000000..f71849f1bb22 --- /dev/null +++ b/apps/server/src/provider/kilo/KiloProcessCleanup.ts @@ -0,0 +1,185 @@ +import * as Crypto from "effect/Crypto"; +import * as Encoding from "effect/Encoding"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; + +export class KiloCleanupError extends Schema.TaggedError()("KiloCleanupError", { + detail: Schema.String, +}) { + override get message() { + return this.detail; + } +} + +const Identity = Schema.Struct({ pid: Schema.Int, startTime: Schema.String }); +const Record = Schema.Struct({ + profile: Schema.String, + // null means observation did not complete; it is not an empty process set. + members: Schema.NullOr(Schema.Array(Identity)), +}); +const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(Record)); +const encode = Schema.encodeEffect(Schema.fromJsonString(Record)); +const gates = new Map }>(); +const failure = () => + new KiloCleanupError({ + detail: + "Kilo process cleanup could not be confirmed. Replacement remains blocked; check process and state-directory access before retrying.", + }); + +/** Linux observation only: not a sandbox, process-tree discovery or atomic signal identity. */ +export const make = Effect.fn("KiloProcessCleanup.make")(function* (input: { + readonly profile: string; + readonly stateDir: string; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + // Create the owned directory before canonicalizing: first-open aliases must use + // the same gate and record namespace as subsequent opens through the real path. + yield* fs.makeDirectory(input.profile, { recursive: true }).pipe(Effect.mapError(failure)); + const profile = yield* fs.realPath(input.profile).pipe(Effect.mapError(failure)); + const crypto = yield* Crypto.Crypto; + const key = yield* crypto + .digest("SHA-256", new TextEncoder().encode(profile)) + .pipe(Effect.map(Encoding.encodeHex), Effect.mapError(failure)); + let gate = gates.get(profile); + if (!gate) { + gate = { lock: Semaphore.makeUnsafe(1), unrecorded: new Set() }; + gates.set(profile, gate); + } + const handoff = gate; + const directory = path.join(input.stateDir, "kilo-cleanup", key); + const observe = (pid: number) => + fs.readFileString(`/proc/${pid}/stat`).pipe( + Effect.flatMap((stat) => { + const fields = stat + .slice(stat.lastIndexOf(")") + 2) + .trim() + .split(/\s+/); + if ( + !stat.startsWith(`${pid} (`) || + !/^[A-Za-z]$/.test(fields[0] ?? "") || + !/^\d+$/.test(fields[2] ?? "") || + !/^\d+$/.test(fields[19] ?? "") + ) + return Effect.fail(failure()); + return Effect.succeed({ + pid, + startTime: fields[19]!, + pgid: Number(fields[2]), + stopped: fields[0] === "Z" || fields[0] === "X" || fields[0] === "x", + }); + }), + Effect.catchTag("PlatformError", (error) => + error.reason._tag === "NotFound" ? Effect.succeed(undefined) : Effect.fail(failure()), + ), + ); + const wait = (members: ReadonlyArray) => + Effect.gen(function* () { + for (;;) { + const statuses = yield* Effect.forEach( + members, + (member) => + observe(member.pid).pipe( + Effect.map( + (current) => + current === undefined || + current.startTime !== member.startTime || + current.stopped, + ), + ), + { concurrency: 8 }, + ); + if (statuses.every(Boolean)) return; + // A bounded condition-driven OS observation, never a fixed cleanup grace assertion. + yield* Effect.sleep("10 millis"); + } + }).pipe(Effect.timeout("2 seconds"), Effect.mapError(failure)); + const snapshot = (pgid: number) => + Effect.gen(function* () { + const names = yield* fs.readDirectory("/proc").pipe(Effect.mapError(failure)); + const observed = yield* Effect.forEach( + names.filter((name) => /^\d+$/.test(name)), + (name) => observe(Number(name)), + { concurrency: 16 }, + ); + return observed.flatMap((item) => (item !== undefined && item.pgid === pgid ? [item] : [])); + }); + // For incomplete snapshots, a successful fresh group scan must prove quiescence. + // This never signals newly discovered PIDs, nor claims to find escaped descendants. + const confirmGroup = (pgid: number) => + snapshot(pgid).pipe( + Effect.flatMap((members) => + members.every((member) => member.stopped) ? Effect.void : Effect.fail(failure()), + ), + ); + const entries = fs.readDirectory(directory).pipe( + Effect.catchTag("PlatformError", (error) => + error.reason._tag === "NotFound" ? Effect.succeed([] as Array) : Effect.fail(error), + ), + Effect.mapError(failure), + ); + const save = (file: string, members: (typeof Record.Type)["members"]) => + Effect.gen(function* () { + const temporary = `${file}.tmp`; + yield* fs + .writeFileString(temporary, yield* encode({ profile, members })) + .pipe(Effect.mapError(failure)); + // Same-directory rename keeps the previous uncertainty record readable if a + // later write is interrupted or partially fails. This is not a power-loss fsync guarantee. + yield* fs.rename(temporary, file).pipe(Effect.mapError(failure)); + }); + const recover = Effect.gen(function* () { + for (const pgid of handoff.unrecorded) { + yield* confirmGroup(pgid); + // A working read is not proof that writes have recovered. + yield* fs.makeDirectory(directory, { recursive: true }).pipe(Effect.mapError(failure)); + const file = path.join(directory, `${pgid}.json`); + yield* save(file, []); + yield* fs.remove(file).pipe(Effect.mapError(failure)); + handoff.unrecorded.delete(pgid); + } + for (const name of yield* entries) { + if (!/^\d+\.json$/.test(name)) continue; + const file = path.join(directory, name); + const record = yield* fs + .readFileString(file) + .pipe(Effect.flatMap(decode), Effect.mapError(failure)); + if (record.profile !== profile) continue; + if (record.members === null) yield* confirmGroup(Number(name.slice(0, -5))); + else yield* wait(record.members); + yield* fs.remove(file).pipe(Effect.mapError(failure)); + } + }); + const verify = (pgid: number, stop: Effect.Effect) => + Effect.gen(function* () { + handoff.unrecorded.add(pgid); + yield* fs.makeDirectory(directory, { recursive: true }).pipe(Effect.mapError(failure)); + const file = path.join(directory, `${pgid}.json`); + // Persist uncertainty before taking a snapshot or delivering any signal. + yield* save(file, null); + handoff.unrecorded.delete(pgid); + const members = (yield* snapshot(pgid)).map(({ pid, startTime }) => ({ pid, startTime })); + yield* save(file, members); + yield* stop; + yield* wait(members); + yield* fs.remove(file).pipe(Effect.mapError(failure)); + }); + return { + // A short start/cleanup gate, not a session-lifetime lock: live sessions may coexist. + withStart: (start: Effect.Effect) => + handoff.lock.withPermit( + recover.pipe(Effect.timeout("5 seconds"), Effect.mapError(failure), Effect.andThen(start)), + ), + verify: (pgid: number, stop: Effect.Effect) => + handoff.lock.withPermit( + verify(pgid, stop).pipe( + Effect.timeout("5 seconds"), + Effect.mapError(failure), + Effect.interruptible, + ), + ), + }; +}); diff --git a/apps/server/src/provider/kilo/KiloRuntime.cleanup.test.ts b/apps/server/src/provider/kilo/KiloRuntime.cleanup.test.ts new file mode 100644 index 000000000000..03dc9b5ce56f --- /dev/null +++ b/apps/server/src/provider/kilo/KiloRuntime.cleanup.test.ts @@ -0,0 +1,241 @@ +// @effect-diagnostics nodeBuiltinImport:off - real CLI lifecycle and spawn observation. +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as NodeChildProcess from "node:child_process"; +import * as NodeURL from "node:url"; +import * as NodeFS from "node:fs"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import * as KiloRuntime from "./KiloRuntime.ts"; + +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const binary = process.env.KILO_BIN; +it.live.skipIf(!binary || HostProcessPlatform.defaultValue() !== "linux")( + "confirms the observed child exit before cleanup returns and a replacement really starts", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-handoff-" }); + const plugin = path.join(root, "fixture.ts"); + const marker = path.join(root, "child"); + const ready = yield* Deferred.make(); + yield* Effect.acquireRelease( + Effect.sync(() => + NodeFS.watch(root, (event, file) => { + if (event === "change" && file === "child") Deferred.doneUnsafe(ready, Effect.void); + }), + ), + (watcher) => Effect.sync(() => watcher.close()), + ); + const childCode = `require('node:fs').writeFileSync(${encodeJson(marker)},String(process.pid));setInterval(()=>{},1000)`; + yield* fs.writeFileString( + plugin, + `import {spawn} from 'node:child_process'; +spawn(${encodeJson(process.execPath)},['-e',${encodeJson(childCode)}],{stdio:'ignore'}); +export const fixture=async()=>({});\n`, + ); + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const replacement = yield* Deferred.make(); + const timeline: Array = []; + let armed = false; + let child = 0; + let identity = ""; + let starts = 0; + const observedFs: FileSystem.FileSystem = { + ...fs, + readDirectory: (directory) => + directory === "/proc" && armed + ? Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen(fs.readDirectory(directory)), + ) + : fs.readDirectory(directory), + readFileString: (file, ...args) => + fs.readFileString(file, ...args).pipe( + Effect.tap((stat) => + Effect.sync(() => { + if (!armed || file !== `/proc/${child}/stat`) return; + const fields = stat + .slice(stat.lastIndexOf(")") + 2) + .trim() + .split(/\s+/); + assert.equal(fields[19], identity); + if (fields[0] === "Z" || fields[0] === "X") timeline.push("child-exit-confirmed"); + }), + ), + Effect.tapError((error) => + Effect.sync(() => { + if (armed && file === `/proc/${child}/stat` && error.reason._tag === "NotFound") + timeline.push("child-exit-confirmed"); + }), + ), + ), + }; + const observedSpawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + starts++; + if (starts === 2) { + timeline.push("replacement-spawn"); + yield* Deferred.succeed(replacement, undefined); + } + return yield* spawner.spawn(command); + }), + ); + yield* Effect.gen(function* () { + const runtime = yield* KiloRuntime.make({ + instanceId: "handoff", + binaryPath: binary!, + profileDirectory: path.join(root, "profile"), + environment: { + PATH: process.env.PATH, + HTTP_PROXY: process.env.HTTP_PROXY, + HTTPS_PROXY: process.env.HTTPS_PROXY, + NO_PROXY: process.env.NO_PROXY, + NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS, + KILO_DISABLE_PROJECT_CONFIG: "1", + HOME: root, + npm_config_offline: "true", + KILO_DISABLE_MODELS_FETCH: "1", + KILO_DISABLE_DEFAULT_PLUGINS: "1", + KILO_DISABLE_EXTERNAL_SKILLS: "1", + KILO_CONFIG_CONTENT: encodeJson({ plugin: [plugin] }), + }, + }); + const first = yield* runtime.open(root); + yield* first.client.models(); + yield* Deferred.await(ready); + child = Number(yield* fs.readFileString(marker)); + const fields = (yield* fs.readFileString(`/proc/${child}/stat`)) + .split(")") + .at(-1)! + .trim() + .split(/\s+/); + identity = fields[19]!; + assert.notEqual(fields[0], "Z"); + armed = true; + const handoff = yield* first.stop.pipe( + Effect.tap(() => Effect.sync(() => timeline.push("cleanup-return"))), + Effect.andThen(runtime.open(root)), + Effect.forkScoped, + ); + // Old cleanup reaches the real second spawn instead of the observation gate. + const milestone = yield* Effect.raceFirst( + Deferred.await(entered).pipe(Effect.as("observation")), + Deferred.await(replacement).pipe(Effect.as("replacement")), + ); + // Release even when the negative control fails, so no fixture is parked. + yield* Deferred.succeed(release, undefined); + assert.equal(milestone, "observation"); + const second = yield* Fiber.join(handoff); + assert.isTrue(yield* second.isRunning); + assert.equal(starts, 2); + assert.isTrue(timeline.indexOf("child-exit-confirmed") >= 0); + assert.isBelow( + timeline.indexOf("child-exit-confirmed"), + timeline.indexOf("cleanup-return"), + ); + assert.isBelow(timeline.indexOf("cleanup-return"), timeline.indexOf("replacement-spawn")); + armed = false; + yield* second.stop; + }).pipe( + Effect.provideService(FileSystem.FileSystem, observedFs), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, observedSpawner), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + { timeout: 30000 }, +); + +it.live.skipIf(HostProcessPlatform.defaultValue() !== "linux")( + "recovers a persisted cleanup reservation after its writer crashes in another OS process", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-restart-" }); + const launch = (code: string) => + Effect.acquireRelease( + Effect.gen(function* () { + const ready = yield* Deferred.make(); + const exited = yield* Deferred.make(); + let output = ""; + let stderr = ""; + const child = NodeChildProcess.spawn( + process.execPath, + ["--input-type=module", "-e", code], + { + cwd: NodeURL.fileURLToPath(new URL("../../../", import.meta.url)), + detached: true, + stdio: ["ignore", "pipe", "pipe"], + env: { PATH: process.env.PATH }, + }, + ); + child.stdout.on("data", (chunk) => { + output += String(chunk); + if (output.includes("ready")) Deferred.doneUnsafe(ready, Effect.void); + }); + child.stderr.on("data", (chunk) => { + stderr += String(chunk); + }); + child.once("error", (error) => { + Deferred.doneUnsafe(ready, Effect.die(error)); + Deferred.doneUnsafe(exited, Effect.die(error)); + }); + child.once("exit", (code) => { + Deferred.doneUnsafe(ready, Effect.die(`fixture exited before readiness: ${stderr}`)); + Deferred.doneUnsafe(exited, Effect.succeed(code)); + }); + return { + child, + ready: Deferred.await(ready), + exited: Deferred.await(exited), + output: () => output, + }; + }), + ({ child, exited }) => + Effect.sync(() => { + child.kill("SIGKILL"); + }).pipe(Effect.andThen(exited), Effect.asVoid), + ); + const member = yield* launch("process.stdout.write('ready');setInterval(()=>{},1000)"); + yield* member.ready; + const imports = `import * as Effect from 'effect/Effect'; +import * as NodeServices from '@effect/platform-node/NodeServices'; +import * as Cleanup from ${encodeJson(new URL("./KiloProcessCleanup.ts", import.meta.url).href)}; +`; + const make = `const cleanup = yield* Cleanup.make({profile:${encodeJson(root)},stateDir:${encodeJson(root)}});`; + const writer = yield* launch(`${imports} +await Effect.runPromise(Effect.gen(function*(){${make} +yield* cleanup.verify(${member.child.pid},Effect.sync(()=>process.stdout.write('ready')).pipe(Effect.andThen(Effect.never))); +}).pipe(Effect.provide(NodeServices.layer)));`); + yield* writer.ready; // The real file was saved before verify enters its stop action. + writer.child.kill("SIGKILL"); + yield* writer.exited; + const attempt = `${imports} +await Effect.runPromise(Effect.gen(function*(){${make} +yield* cleanup.withStart(Effect.sync(()=>process.stdout.write('spawn'))); +}).pipe(Effect.provide(NodeServices.layer),Effect.catch(()=>Effect.sync(()=>{process.exitCode=23;}))));`; + const blocked = yield* launch(attempt); + assert.equal(yield* blocked.exited, 23); + assert.equal(blocked.output(), ""); + const [key] = yield* fs.readDirectory(path.join(root, "kilo-cleanup")); + assert.deepEqual(yield* fs.readDirectory(path.join(root, "kilo-cleanup", key!)), [ + `${member.child.pid}.json`, + ]); + member.child.kill("SIGKILL"); + yield* member.exited; // This parent reaps its own child, rather than inferring exit from kill success. + const recovered = yield* launch(attempt); + assert.equal(yield* recovered.exited, 0); + assert.equal(recovered.output(), "spawn"); + assert.deepEqual(yield* fs.readDirectory(path.join(root, "kilo-cleanup", key!)), []); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + { timeout: 15000 }, +); diff --git a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts index 0f586f907918..0731ee7b7b3b 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts @@ -95,17 +95,47 @@ describe.skipIf(!binary)("KiloRuntime native lifecycle", () => { Number(yield* fs.readFileString(marker)), Number(yield* fs.readFileString(explicitMarker)), ]; - const running = (pid: number) => + const observe = (pid: number) => fs.readFileString(`/proc/${pid}/stat`).pipe( - Effect.map((stat) => !stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")), - Effect.orElseSucceed(() => false), + Effect.map((stat) => { + const fields = stat + .slice(stat.lastIndexOf(")") + 2) + .trim() + .split(/\s+/); + assert.match(fields[19]!, /^\d+$/); + return { startTime: fields[19]!, state: fields[0]! }; + }), + Effect.catchTag("PlatformError", (error) => + error.reason._tag === "NotFound" ? Effect.succeed(undefined) : Effect.fail(error), + ), ); - for (const pid of descendants) assert.isTrue(yield* running(pid)); - // Kill only the recorded owned leader while its session is idle. Exit - // observation must clean up descendants without an active-turn error. + const identities = yield* Effect.forEach(descendants, (pid) => + Effect.gen(function* () { + const observed = yield* observe(pid); + assert.isDefined(observed); + assert.notEqual(observed!.state, "Z"); + return { pid, startTime: observed!.startTime }; + }), + ); + // This test proves eventual cleanup of these fixture children, not the + // ordering of replacement. The handoff test verifies that separately. process.kill(recorded.pgid, "SIGKILL"); yield* connection.exitCode; - for (const pid of descendants) assert.isFalse(yield* running(pid)); + yield* Effect.forEach(identities, (identity) => + Effect.gen(function* () { + for (;;) { + const current = yield* observe(identity.pid); + if ( + current === undefined || + current.startTime !== identity.startTime || + current.state === "Z" || + current.state === "X" + ) + return; + yield* Effect.sleep("10 millis"); + } + }).pipe(Effect.timeout("2 seconds")), + ); } }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), { timeout: 30000 }, diff --git a/apps/server/src/provider/kilo/KiloRuntime.ts b/apps/server/src/provider/kilo/KiloRuntime.ts index 912cfdac920d..5ec078484776 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.ts @@ -14,6 +14,7 @@ import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { resolveSpawnCommand } from "@t3tools/shared/shell"; import { signalProcessGroup } from "../../process/processGroup.ts"; +import * as KiloProcessCleanup from "./KiloProcessCleanup.ts"; import * as KiloSessionClient from "./KiloSessionClient.ts"; import * as ServerLedger from "../OpenCodeServerLedger.ts"; @@ -104,6 +105,13 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { const fail = (operation: string, detail: string) => (cause: unknown) => new KiloRuntimeError({ operation, detail, cause }); const profile = path.resolve(input.profileDirectory); + const processCleanup = + platform === "linux" + ? yield* KiloProcessCleanup.make({ + profile, + stateDir: input.processStateDirectory ?? path.join(profile, "t3-processes"), + }).pipe(Effect.mapError(fail("cleanup", "Could not prepare Kilo process cleanup."))) + : undefined; const ledger = yield* ServerLedger.make({ stateDir: input.processStateDirectory ?? path.join(profile, "t3-processes"), }); @@ -187,7 +195,7 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { ); // Forget only after the owned group is stopped, including failed readiness. const ledgerScope = yield* Scope.fork(scope); - const child = yield* spawner + const spawn = spawner .spawn( ChildProcess.make(command.command, command.args, { cwd: directory, @@ -198,33 +206,62 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { }), ) .pipe(Effect.mapError(fail("spawn", "Could not start Kilo. Check the binary path."))); + const child = yield* (processCleanup ? processCleanup.withStart(spawn) : spawn).pipe( + Effect.mapError((cause) => + isRuntimeError(cause) + ? cause + : fail( + "cleanup", + "Kilo could not start while previous process cleanup is unconfirmed.", + )(cause), + ), + ); // Only this captured process group is signalled. No process-name matching. + const signal = Effect.uninterruptible( + platform === "win32" + ? child.kill({ killSignal: "SIGTERM", forceKillAfter: "1 second" }).pipe(Effect.ignore) + : Effect.sync(() => { + try { + signalProcessGroup(Number(child.pid), "SIGTERM"); + } catch { + /* already exited */ + } + }).pipe( + Effect.andThen( + child.exitCode.pipe(Effect.timeoutOption("1 second"), Effect.ignore), + ), + Effect.andThen( + Effect.sync(() => { + try { + signalProcessGroup(Number(child.pid), "SIGKILL"); + } catch { + /* already exited */ + } + }), + ), + ), + ); + let verified = false; const cleanup = yield* Effect.cached( - Effect.uninterruptible( - platform === "win32" - ? child - .kill({ killSignal: "SIGTERM", forceKillAfter: "1 second" }) - .pipe(Effect.ignore) - : Effect.sync(() => { - try { - signalProcessGroup(Number(child.pid), "SIGTERM"); - } catch { - /* already exited */ - } - }).pipe( - Effect.andThen( - child.exitCode.pipe(Effect.timeoutOption("1 second"), Effect.ignore), - ), - Effect.andThen( - Effect.sync(() => { - try { - signalProcessGroup(Number(child.pid), "SIGKILL"); - } catch { - /* already exited */ - } - }), - ), + (processCleanup + ? processCleanup.verify( + Number(child.pid), + signal.pipe( + // Await/reap our own child as well as observing non-child members. + // This wait stays inside the bounded, interruptible verification. + // The Node spawner reports signal termination as an exitCode + // error after the actual exit event. PID observation still follows. + Effect.andThen(child.exitCode.pipe(Effect.ignore)), ), + ) + : signal + ).pipe( + Effect.tap(() => + Effect.sync(() => { + verified = true; + }), + ), + Effect.orDie, ), ); yield* Effect.addFinalizer(() => cleanup); @@ -233,7 +270,10 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { port: 0, args: ["serve", "--hostname=127.0.0.1", "--port=0"], }); - yield* Scope.addFinalizer(ledgerScope, forget); + yield* Scope.addFinalizer( + ledgerScope, + Effect.suspend(() => (verified ? forget : Effect.void)), + ); const guard = checkAuth.pipe(Effect.onError(() => cleanup)); // Observe idle or in-flight account replacement as well as request boundaries. // Never read credential files once per SSE event. diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md index 825f48d287ef..78815c157b21 100644 --- a/docs/user/providers-kilo.md +++ b/docs/user/providers-kilo.md @@ -32,6 +32,26 @@ records are recovered only if that original profile is reopened. Deleting or mov T3's own state directory can lose cleanup records; T3 never guesses ownership from a process name. +On Linux, normal cleanup snapshots the current process group before signalling. +T3 waits for each recorded PID/start-time identity to disappear or reach a +non-executing zombie/dead state before cleanup returns. Its owned child exit is +reaped through the process spawner. Within one T3 server, a per-profile gate covers cleanup and this +verification, and is checked before another local process starts; it does not +prevent concurrent live sessions or lock out another T3 server. Observation errors and timeouts leave a pending +record and block new starts for that profile. Retry can clear it only after a +successful exit observation, not merely because time elapsed. Incomplete snapshots +require a complete fresh scan showing no executing members in the original group. + +This is not a supervisor or sandbox. Enumeration is not atomic: newly forked or +escaped descendants, process-group changes and PID reuse between a native signal +check and delivery are not fully contained. The stronger exit observation is +Linux-only; macOS and Windows retain their existing cleanup behavior. Persisted +pending records survive an ordinary T3 restart if the state directory remains +intact. Failed initial writes only retain uncertainty in memory; power loss, +corrupted/deleted records and crashes before cleanup starts do not gain a stronger +guarantee than the existing orphan reaper. T3 does not rewrite native MCP or plugin +configuration to enforce this lifecycle boundary. + | Capability | Local Kilo | Kilo Cloud | | ------------------------------------------ | -------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | | Prompts and follow-up | Native streaming, tools and reasoning | Full access; history updates, no token-streaming claim | From 567b771c57696c4baf608284a6c677279091387e Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Mon, 5 Oct 2026 05:44:52 +0000 Subject: [PATCH 27/44] fix(kilo): keep explicit MCP cloud targets off local workspaces --- .../mcp/toolkits/worktree/handlers.test.ts | 149 ++++++++++++++++++ .../src/mcp/toolkits/worktree/handlers.ts | 23 ++- .../server/src/mcp/toolkits/worktree/tools.ts | 2 + .../RunExecutionService.test.ts | 19 ++- 4 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 apps/server/src/mcp/toolkits/worktree/handlers.test.ts diff --git a/apps/server/src/mcp/toolkits/worktree/handlers.test.ts b/apps/server/src/mcp/toolkits/worktree/handlers.test.ts new file mode 100644 index 000000000000..672c86f2291c --- /dev/null +++ b/apps/server/src/mcp/toolkits/worktree/handlers.test.ts @@ -0,0 +1,149 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import { expect, it } from "@effect/vitest"; +import { + EnvironmentId, + ProviderDriverKind, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { McpSchema, McpServer } from "effect/unstable/ai"; +import * as GitWorkflow from "../../../git/GitWorkflowService.ts"; +import * as ProviderAdapterRegistry from "../../../orchestration-v2/ProviderAdapterRegistry.ts"; +import * as ThreadManagement from "../../../orchestration-v2/ThreadManagementService.ts"; +import * as Project from "../../../project/ProjectService.ts"; +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import * as WorktreeMcpService from "../../WorktreeMcpService.ts"; +import { WorktreeToolkitHandlersLive } from "./handlers.ts"; +import { WorktreeToolkit } from "./tools.ts"; + +it.effect.each([ + { persisted: "kilo-cloud", configured: "kilo", cwd: null, explicit: true, allowed: false }, + { persisted: "kilo-cloud", configured: "missing", cwd: null, explicit: true, allowed: false }, + { persisted: null, configured: "kilo-cloud", cwd: null, explicit: true, allowed: false }, + { + persisted: "kilo", + configured: "kilo-cloud", + cwd: "/local/worktree", + explicit: true, + allowed: true, + }, + { persisted: null, configured: "kilo", cwd: null, explicit: true, allowed: true }, + { persisted: null, configured: "kilo", cwd: null, explicit: false, allowed: true }, + { persisted: null, configured: "missing", cwd: null, explicit: true, allowed: false }, +])("keeps refs on the target workspace: %j", (test) => + Effect.gen(function* () { + const targetId = ThreadId.make("target-thread"); + const instanceId = ProviderInstanceId.make("target-account"); + const gitCalls: string[] = []; + const thread = { + id: targetId, + projectId: "target-project", + providerInstanceId: instanceId, + activeProviderThreadId: test.persisted ? "native-binding" : null, + worktreePath: test.cwd, + deletedAt: null, + }; + const dependencies = Layer.mergeAll( + Layer.mock(ThreadManagement.ThreadManagementService)({ + getThreadShell: () => Effect.succeed(thread as never), + getProjectThreadRecords: () => + Effect.succeed({ + thread, + providerThreads: test.persisted + ? [{ id: "native-binding", driver: test.persisted }] + : [], + } as never), + }), + Layer.mock(ProviderAdapterRegistry.ProviderAdapterRegistryV2)({ + getMetadata: (id) => { + expect(id).toBe(instanceId); + return test.configured === "missing" + ? Effect.fail( + new ProviderAdapterRegistry.ProviderAdapterRegistryLookupError({ instanceId: id }), + ) + : Effect.succeed({ driver: ProviderDriverKind.make(test.configured) } as never); + }, + }), + Layer.mock(Project.ProjectService)({ + getById: () => Effect.succeed(Option.some({ workspaceRoot: "/local/project" } as never)), + }), + Layer.mock(GitWorkflow.GitWorkflowService)({ + listRefs: ({ cwd }) => { + gitCalls.push(cwd); + return Effect.succeed({ + refs: [], + nextCursor: null, + isRepo: true, + hasPrimaryRemote: false, + totalCount: 0, + }); + }, + }), + Layer.mock(WorktreeMcpService.WorktreeMcpService)({}), + ); + const result = yield* Effect.gen(function* () { + const server = yield* McpServer.McpServer; + return yield* server + .callTool({ + name: "t3_worktree_list", + arguments: test.explicit ? { threadId: targetId } : {}, + }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, { + environmentId: EnvironmentId.make("environment"), + requestNamespace: "local-caller", + thread: test.explicit + ? undefined + : { + threadId: targetId, + providerInstanceId: instanceId, + providerSessionId: "local-session", + }, + client: test.explicit + ? { sessionId: "client", label: "fixture", runtimeModeCeiling: "full-access" } + : undefined, + capabilities: new Set([ + "orchestration", + "worktree", + ]), + issuedAt: 0, + }), + Effect.provideService(McpSchema.McpServerClient, { + clientId: 1, + clientCapabilities: {}, + initializePayload: { + protocolVersion: "2024-11-05", + capabilities: {}, + clientInfo: { name: "fixture", version: "1" }, + }, + getClient: Effect.die("unused"), + clientInfo: { name: "fixture", version: "1" }, + protocolVersion: "2024-11-05", + }), + ); + }).pipe( + Effect.provide( + McpServer.toolkit(WorktreeToolkit).pipe( + Layer.provide(WorktreeToolkitHandlersLive), + Layer.provideMerge(McpServer.McpServer.layer), + Layer.provide(NodeCrypto.layer), + Layer.provide(dependencies), + ), + ), + ); + expect(gitCalls).toEqual(test.allowed ? [test.cwd ?? "/local/project"] : []); + if (test.allowed) { + expect(result.isError).toBe(false); + } else { + expect(result.structuredContent).toMatchObject({ + code: + test.persisted === null && test.configured === "missing" + ? "orchestration_error" + : "capability_denied", + }); + } + }), +); diff --git a/apps/server/src/mcp/toolkits/worktree/handlers.ts b/apps/server/src/mcp/toolkits/worktree/handlers.ts index dbb7c59f4fb7..fdd7745f6880 100644 --- a/apps/server/src/mcp/toolkits/worktree/handlers.ts +++ b/apps/server/src/mcp/toolkits/worktree/handlers.ts @@ -1,6 +1,7 @@ import { OrchestratorMcpFailure } from "@t3tools/contracts"; import * as Option from "effect/Option"; import * as GitWorkflow from "../../../git/GitWorkflowService.ts"; +import * as ProviderAdapterRegistry from "../../../orchestration-v2/ProviderAdapterRegistry.ts"; import * as Project from "../../../project/ProjectService.ts"; import { readThread, unavailable } from "../../threadAccess.ts"; import * as Effect from "effect/Effect"; @@ -20,8 +21,26 @@ const handlers = { }); const { threadId, ...refs } = input; const { - projection: { thread }, - } = yield* readThread(threadId); + projection: { thread, providerThreads }, + } = yield* readThread(threadId, ["providerThreads"]); + // A cloud thread's project is a UI grouping, not its remote checkout. + // Prefer its durable binding over a provider configuration changed since launch. + const binding = providerThreads.find((item) => item.id === thread.activeProviderThreadId); + const adapters = yield* ProviderAdapterRegistry.ProviderAdapterRegistryV2; + const driver = + binding?.driver ?? + (yield* ( + adapters.getMetadata + ? adapters + .getMetadata(thread.providerInstanceId) + .pipe(Effect.map((adapter) => adapter.driver)) + : adapters.get(thread.providerInstanceId).pipe(Effect.map((adapter) => adapter.driver)) + ).pipe(Effect.mapError(unavailable))); + if (driver === "kilo-cloud") + return yield* new OrchestratorMcpFailure({ + code: "capability_denied", + message: "Kilo Cloud has no local workspace refs. Inspect its remote repository instead.", + }); const projects = yield* Project.ProjectService; const project = yield* projects.getById(thread.projectId).pipe(Effect.mapError(unavailable)); if (Option.isNone(project)) diff --git a/apps/server/src/mcp/toolkits/worktree/tools.ts b/apps/server/src/mcp/toolkits/worktree/tools.ts index 373be5d3e46d..240d911d5a33 100644 --- a/apps/server/src/mcp/toolkits/worktree/tools.ts +++ b/apps/server/src/mcp/toolkits/worktree/tools.ts @@ -10,6 +10,7 @@ import { } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; import * as GitWorkflowService from "../../../git/GitWorkflowService.ts"; +import * as ProviderAdapterRegistry from "../../../orchestration-v2/ProviderAdapterRegistry.ts"; import * as ProjectService from "../../../project/ProjectService.ts"; import * as ThreadManagementService from "../../../orchestration-v2/ThreadManagementService.ts"; import { Tool, Toolkit } from "effect/unstable/ai"; @@ -74,6 +75,7 @@ const WorktreeListTool = Tool.make("t3_worktree_list", { ThreadManagementService.ThreadManagementService, ProjectService.ProjectService, GitWorkflowService.GitWorkflowService, + ProviderAdapterRegistry.ProviderAdapterRegistryV2, ], }) .annotate(Tool.Readonly, true) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 8a8a194b6c6b..8b92a55378aa 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -3413,18 +3413,30 @@ it.effect( () => Effect.gen(function* () { for (let attempt = 0; attempt < 3; attempt++) { + const nativeThreadHasTurns = [undefined, false, true][attempt]; + const delivered: Array<{ + reattach: boolean | undefined; + nativeThreadHasTurns: boolean | undefined; + }> = []; const started = yield* Deferred.make(); const stopped = yield* Deferred.make(); const result = yield* captureRootRunTermination({ key: `cloud-reattach-failure-${attempt}`, cloudReattach: true, + ...(nativeThreadHasTurns === undefined ? {} : { nativeThreadHasTurns }), shouldFinalizeRun: () => Effect.succeed(true), events: () => Stream.unwrap(Deferred.succeed(started, undefined).pipe(Effect.as(Stream.never))).pipe( Stream.ensuring(Deferred.succeed(stopped, undefined)), ), startTurn: (input) => - Deferred.await(started).pipe( + Effect.sync(() => + delivered.push({ + reattach: input.reattach, + nativeThreadHasTurns: input.nativeThreadHasTurns, + }), + ).pipe( + Effect.andThen(Deferred.await(started)), Effect.andThen( Effect.fail( new ProviderAdapterTurnStartError({ @@ -3439,6 +3451,7 @@ it.effect( ), }); yield* Deferred.await(stopped); + assert.deepEqual(delivered, [{ reattach: true, nativeThreadHasTurns }]); assert.isTrue(result.startFailed); assert.deepEqual(result.written, []); assert.deepEqual(result.observed, []); @@ -3511,6 +3524,7 @@ it.effect.each([true, false])( function captureRootRunTermination(input: { readonly key: string; readonly cloudReattach?: boolean; + readonly nativeThreadHasTurns?: boolean; readonly checkpointFilesystem?: boolean; readonly shouldFinalizeRun: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; @@ -3600,6 +3614,9 @@ function captureRootRunTermination(input: { appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${input.key}`), reattach: input.cloudReattach ?? false, + ...(input.nativeThreadHasTurns === undefined + ? {} + : { nativeThreadHasTurns: input.nativeThreadHasTurns }), session: { driver: input.cloudReattach ? ProviderDriverKind.make("kilo-cloud") : driver, providerSession: { From 74f003dcd216c36aee83e300a66c7ebecc00936b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 15:56:14 +0000 Subject: [PATCH 28/44] refactor(kilo): trim the provider to the code it runs - Drop PR-only tooling: the UI evidence and stream benchmark scripts, and the paid or evidence-gated live tests. KiloAdapterV2.live.test.ts stays as the one live test, like the other providers. - Own the local Kilo server the way OpenCode does: OpenCodeServerLedger records it and the boot reaper cleans up after a crash. This removes the Linux-only KiloProcessCleanup layer, the per-runtime reapers, and the ledger changes that only they needed. - Remove the unused KiloCloudClient (superseded by KiloCloudWebClient); its error type moves to KiloCloudError.ts. - Share the permission/question presentation and reply mapping between the local and cloud adapters, and the conversation fork used by rewind and fork. - Drop the unused session revert method and username option. - Rewrite the Kilo user guide in product terms and link it from the docs index and provider setup. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01T3SNABJJ9aaJoqFLNyVvSG --- apps/desktop/scripts/kilo-ui-evidence.mjs | 308 ------------ apps/server/scripts/kilo-stream-benchmark.mjs | 135 ------ .../Adapters/KiloAdapterV2.live.test.ts | 1 + .../Adapters/KiloAdapterV2.ts | 322 ++++++------ .../Adapters/KiloCloudAdapterV2.live.test.ts | 459 ------------------ .../Adapters/KiloCloudAdapterV2.test.ts | 1 + .../Adapters/KiloCloudAdapterV2.ts | 146 ++---- .../src/provider/OpenCodeServerLedger.test.ts | 116 ----- .../src/provider/OpenCodeServerLedger.ts | 45 +- .../src/provider/kilo/KiloCloudAccount.ts | 2 +- .../src/provider/kilo/KiloCloudClient.test.ts | 152 ------ .../src/provider/kilo/KiloCloudClient.ts | 218 --------- .../src/provider/kilo/KiloCloudError.ts | 18 + .../kilo/KiloCloudWebClient.live.test.ts | 44 -- .../provider/kilo/KiloCloudWebClient.test.ts | 2 +- .../src/provider/kilo/KiloCloudWebClient.ts | 2 +- .../provider/kilo/KiloProcessCleanup.test.ts | 391 --------------- .../src/provider/kilo/KiloProcessCleanup.ts | 185 ------- .../provider/kilo/KiloRuntime.cleanup.test.ts | 241 --------- .../kilo/KiloRuntime.crash.fixture.mjs | 44 -- .../provider/kilo/KiloRuntime.live.test.ts | 435 ----------------- apps/server/src/provider/kilo/KiloRuntime.ts | 67 +-- .../kilo/KiloSessionClient.live.test.ts | 271 ----------- .../src/provider/kilo/KiloSessionClient.ts | 52 +- docs/README.md | 2 +- docs/user/install.md | 2 +- docs/user/providers-kilo.md | 192 +++----- knip.jsonc | 2 - 28 files changed, 334 insertions(+), 3521 deletions(-) delete mode 100644 apps/desktop/scripts/kilo-ui-evidence.mjs delete mode 100644 apps/server/scripts/kilo-stream-benchmark.mjs delete mode 100644 apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.live.test.ts delete mode 100644 apps/server/src/provider/kilo/KiloCloudClient.test.ts delete mode 100644 apps/server/src/provider/kilo/KiloCloudClient.ts create mode 100644 apps/server/src/provider/kilo/KiloCloudError.ts delete mode 100644 apps/server/src/provider/kilo/KiloCloudWebClient.live.test.ts delete mode 100644 apps/server/src/provider/kilo/KiloProcessCleanup.test.ts delete mode 100644 apps/server/src/provider/kilo/KiloProcessCleanup.ts delete mode 100644 apps/server/src/provider/kilo/KiloRuntime.cleanup.test.ts delete mode 100644 apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs delete mode 100644 apps/server/src/provider/kilo/KiloRuntime.live.test.ts delete mode 100644 apps/server/src/provider/kilo/KiloSessionClient.live.test.ts diff --git a/apps/desktop/scripts/kilo-ui-evidence.mjs b/apps/desktop/scripts/kilo-ui-evidence.mjs deleted file mode 100644 index 94e82c7a9a09..000000000000 --- a/apps/desktop/scripts/kilo-ui-evidence.mjs +++ /dev/null @@ -1,308 +0,0 @@ -// Real web UI + Orchestrator + pinned Kilo CLI. All inference stays on this loopback fixture. -// KILO_BIN=/path/to/kilo KILO_EVIDENCE_DIR=/tmp/evidence node apps/desktop/scripts/kilo-ui-evidence.mjs -import * as NodeFSP from "node:fs/promises"; -import * as NodePath from "node:path"; -import * as NodeOS from "node:os"; -import * as NodeHttp from "node:http"; -import * as NodeEvents from "node:events"; -import * as NodeChildProcess from "node:child_process"; -import * as NodeUtil from "node:util"; -import { chromium } from "playwright-core"; - -if (!process.env.KILO_BIN) throw new Error("KILO_BIN must point to the pinned local CLI"); -if (process.env.KILO_CLOUD_TEST_PROFILE && !process.env.KILO_CLOUD_TEST_REPOSITORY) { - throw new Error( - "KILO_CLOUD_TEST_REPOSITORY is required for an explicitly authorized live capture", - ); -} -const root = NodePath.resolve(import.meta.dirname, "../../.."); -const temporary = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-kilo-ui-")); -const evidence = process.env.KILO_EVIDENCE_DIR ?? NodePath.join(temporary, "evidence"); -const state = NodePath.join(temporary, "state"); -const workspace = NodePath.join(temporary, "kilo-ui-workspace"); -await Promise.all([ - NodeFSP.mkdir(evidence, { recursive: true }), - NodeFSP.mkdir(workspace), - NodeFSP.mkdir(NodePath.join(state, "userdata"), { recursive: true }), -]); -const execFile = NodeUtil.promisify(NodeChildProcess.execFile); -await NodeFSP.writeFile(NodePath.join(workspace, "README.md"), "Local Kilo UI fixture\n"); -for (const args of [ - ["init"], - ["add", "."], - [ - "-c", - "user.name=Fixture", - "-c", - "user.email=fixture@example.invalid", - "commit", - "-m", - "fixture", - ], -]) - await execFile("git", args, { cwd: workspace }); -let requests = 0; -const answer = - "Kilo local integration succeeded. This response came from the isolated local fixture."; -const model = NodeHttp.createServer((req, res) => { - let body = ""; - req.on("data", (chunk) => { - body += chunk; - }); - req.on("end", () => { - requests++; - const data = JSON.parse(body); - const prompt = JSON.stringify(data.messages ?? []); - const text = - prompt.includes("title") && prompt.includes("JSON") - ? '{"title":"Kilo local verification"}' - : answer; - res.writeHead(200, { "Content-Type": data.stream ? "text/event-stream" : "application/json" }); - if (!data.stream) { - res.end( - JSON.stringify({ - id: "local", - object: "chat.completion", - created: 0, - model: "test", - choices: [ - { index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }, - ], - }), - ); - return; - } - for (const chunk of text.match(/.{1,16}/g)) - res.write( - `data: ${JSON.stringify({ - id: "local", - object: "chat.completion.chunk", - created: 0, - model: "test", - choices: [{ index: 0, delta: { content: chunk }, finish_reason: null }], - })}\n\n`, - ); - res.end( - `data: ${JSON.stringify({ - id: "local", - object: "chat.completion.chunk", - created: 0, - model: "test", - choices: [{ index: 0, delta: {}, finish_reason: "stop" }], - usage: { prompt_tokens: 12, completion_tokens: 10, total_tokens: 22 }, - })}\n\ndata: [DONE]\n\n`, - ); - }); -}); -model.listen(0, "127.0.0.1"); -await NodeEvents.once(model, "listening"); -const config = { - model: "fixture/test", - small_model: "fixture/test", - plugin: [], - enabled_providers: ["fixture"], - provider: { - fixture: { - npm: "@ai-sdk/openai-compatible", - name: "Local fixture", - options: { baseURL: `http://127.0.0.1:${model.address().port}/v1` }, - models: { test: { name: "Local fixture", limit: { context: 10000, output: 1000 } } }, - }, - }, -}; -await NodeFSP.writeFile( - NodePath.join(state, "userdata/settings.json"), - JSON.stringify({ - providers: Object.fromEntries( - ["codex", "claudeAgent", "cursor", "grok", "opencode", "antigravity", "pi"].map((name) => [ - name, - { enabled: false }, - ]), - ), - providerInstances: { - kiloCloud: { - driver: "kilo-cloud", - displayName: "Kilo Cloud", - enabled: !!process.env.KILO_CLOUD_TEST_PROFILE, - config: { - enabled: !!process.env.KILO_CLOUD_TEST_PROFILE, - profileDirectory: process.env.KILO_CLOUD_TEST_PROFILE ?? "", - repository: process.env.KILO_CLOUD_TEST_REPOSITORY ?? "synthetic/cloud-demo", - branch: "main", - model: "deepseek/deepseek-v4.1-flash", - cloudConsent: !!process.env.KILO_CLOUD_TEST_PROFILE, - }, - }, - kilo: { - driver: "kilo", - displayName: "Kilo", - enabled: true, - config: { binaryPath: process.env.KILO_BIN, accountId: "ui-fixture" }, - environment: [ - { name: "HOME", value: temporary }, - { name: "KILO_CONFIG_CONTENT", value: JSON.stringify(config) }, - ...[ - "KILO_DISABLE_MODELS_FETCH", - "KILO_DISABLE_DEFAULT_PLUGINS", - "KILO_DISABLE_EXTERNAL_SKILLS", - "KILO_DISABLE_PROJECT_CONFIG", - ].map((name) => ({ name, value: "1" })), - ], - }, - }, - textGenerationModelSelection: { instanceId: "kilo", model: "fixture/test", options: [] }, - }), -); -const child = NodeChildProcess.spawn("vp", ["run", "dev", "--home-dir", state], { - cwd: root, - detached: true, - stdio: ["ignore", "pipe", "pipe"], -}); -let browser; -let page; -try { - const pair = await new Promise((resolve, reject) => { - const timeout = setTimeout(() => reject(new Error("Isolated T3 did not become ready")), 120000); - let output = ""; - const read = (chunk) => { - output = (output + chunk).slice(-50000); - // Startup output may contain ANSI color escapes immediately after the URL. - // oxlint-disable-next-line no-control-regex - const match = /pairingUrl:\s*(http[^\s\x1b]+)/.exec(output); - if (match) { - clearTimeout(timeout); - resolve(match[1]); - } - }; - child.stdout.on("data", read); - child.stderr.on("data", read); - child.once("error", (error) => { - clearTimeout(timeout); - reject(error); - }); - child.once("exit", (code) => { - clearTimeout(timeout); - reject(new Error(`T3 exited with ${code}`)); - }); - }); - browser = await chromium.launch({ - headless: true, - ...(process.env.CHROMIUM_PATH ? { executablePath: process.env.CHROMIUM_PATH } : {}), - args: ["--no-sandbox"], - }); - const context = await browser.newContext({ - viewport: { width: 1440, height: 1000 }, - recordVideo: { dir: evidence, size: { width: 1440, height: 1000 } }, - }); - page = await context.newPage(); - page.setDefaultTimeout(45000); - await page.goto(pair); - await page.getByRole("button", { name: "Add project", exact: true }).click(); - await page.getByText("Local folder", { exact: true }).click(); - await page.getByPlaceholder("Enter path (e.g. ~/projects/my-app)").fill(workspace); - await page.getByPlaceholder("Enter path (e.g. ~/projects/my-app)").press("Enter"); - if (process.env.KILO_CLOUD_TEST_PROFILE) { - await page.locator("[data-chat-provider-model-picker-label]").click(); - await page.getByPlaceholder("Search models...").fill("deepseek-v4.1-flash"); - await page.getByText("deepseek/deepseek-v4.1-flash", { exact: true }).last().click(); - await page - .getByText("Closing T3 does not stop remote work or billing.", { exact: false }) - .waitFor(); - await page.getByRole("button", { name: "Unknown", exact: true }).click(); - await page.getByRole("menuitemradio", { name: /^Low/ }).click(); - await page.getByRole("button", { name: "Low", exact: true }).waitFor(); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "cloud-before-send.png"), - }); - } - await page.locator("[data-chat-provider-model-picker-label]").click(); - await page.getByPlaceholder("Search models...").fill("Local fixture"); - await page.getByText("Local fixture", { exact: true }).last().click(); - await page.getByRole("button", { name: "Local fixture", exact: true }).waitFor(); - await page.locator("[contenteditable=true]").fill("Kilo local integration: say hello."); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "before-send.png"), - }); - await page.getByRole("button", { name: "Submit message", exact: true }).click(); - await page.getByText(answer, { exact: true }).waitFor({ timeout: 60000 }); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "streamed-answer.png"), - }); - await page.getByRole("button", { name: "Submit message", exact: true }).waitFor(); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "completed-answer.png"), - }); - console.log("Local native answer rendered; opening provider settings."); - await page.getByRole("button", { name: "Settings", exact: true }).click(); - await page.waitForURL("**/settings/general*"); - await page.getByText("Restore device defaults", { exact: true }).waitFor(); - await page.getByRole("button", { name: "Providers", exact: true }).click(); - await page.waitForURL("**/settings/providers*"); - await page.getByRole("button", { name: "Add provider", exact: true }).waitFor(); - await page.getByRole("button", { name: "Select Kilo", exact: true }).click(); - await page - .getByText("Native configuration and MCP servers are trusted", { exact: false }) - .first() - .waitFor(); - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "local-trust-settings.png"), - }); - await page.getByRole("button", { name: "Select Kilo Cloud", exact: true }).click(); - if (process.env.KILO_CLOUD_TEST_PROFILE) { - const consent = page.getByRole("switch", { name: "Allow paid cloud execution", exact: true }); - await consent.click(); - await page.waitForFunction( - () => - document - .querySelector('[role="switch"][aria-label="Allow paid cloud execution"]') - ?.getAttribute("aria-checked") === "false", - ); - await consent.click(); - await page.waitForFunction( - () => - document - .querySelector('[role="switch"][aria-label="Allow paid cloud execution"]') - ?.getAttribute("aria-checked") === "true", - ); - } - await page.screenshot({ - animations: "disabled", - path: NodePath.join(evidence, "provider-settings.png"), - }); - await context.close(); - if (!requests) throw new Error("The real CLI did not contact the local inference fixture"); - await NodeFSP.writeFile( - NodePath.join(evidence, "verification.json"), - JSON.stringify( - { - commit: (await execFile("git", ["rev-parse", "HEAD"], { cwd: root })).stdout.trim(), - inferenceRequests: requests, - nativeConfigurationTrusted: true, - inference: "loopback fixture only", - client: "Chromium web", - }, - null, - 2, - ), - ); - console.log("Kilo UI verification passed; screenshots and video saved."); -} catch (error) { - await page?.screenshot({ path: NodePath.join(evidence, "failure.png") }).catch(() => {}); - console.error(String(error).replace(/https?:\/\/[^\s)]+/g, "[local URL]")); - process.exitCode = 1; -} finally { - await browser?.close(); - // This is the process group captured at spawn, never a PID discovered by matching. - try { - process.kill(-child.pid, "SIGTERM"); - } catch { - /* already exited */ - } - model.closeAllConnections(); - await new Promise((resolve) => model.close(resolve)); -} diff --git a/apps/server/scripts/kilo-stream-benchmark.mjs b/apps/server/scripts/kilo-stream-benchmark.mjs deleted file mode 100644 index ef2202276bd5..000000000000 --- a/apps/server/scripts/kilo-stream-benchmark.mjs +++ /dev/null @@ -1,135 +0,0 @@ -// node --expose-gc apps/server/scripts/kilo-stream-benchmark.mjs /absolute/baseline/KiloSessionClient.ts -// Extract the unchanged client with git show, beside a node_modules link to apps/server/node_modules. -// Baseline and current perform the same ownership read, filtering, validation and consumption. -// The raw SDK is diagnostic only: it does less work and is not a regression baseline. -import * as NodeHttp from "node:http"; -import * as NodeEvents from "node:events"; -import * as NodeURL from "node:url"; -import * as NodeOS from "node:os"; -import * as NodeAssert from "node:assert/strict"; -import { createKiloClient } from "@kilocode/sdk/v2"; -import * as Effect from "effect/Effect"; -import * as Stream from "effect/Stream"; -import { make } from "../src/provider/kilo/KiloSessionClient.ts"; - -if (!process.argv[2] || !global.gc) throw new Error("Pass a baseline path and --expose-gc"); -const baseline = await import(NodeURL.pathToFileURL(process.argv[2]).href); -const ref = { instanceId: "bench", directory: "/bench", sessionId: "ses_bench" }; -let workload = { count: 10000, intervalMs: 0 }; -const server = NodeHttp.createServer((req, res) => { - if (!req.url.startsWith("/event")) { - res.writeHead(200, { "content-type": "application/json" }); - res.end( - JSON.stringify( - req.url.startsWith("/global/health") - ? { healthy: true, version: "7.8.3" } - : { id: ref.sessionId, directory: ref.directory }, - ), - ); - return; - } - res.writeHead(200, { "content-type": "text/event-stream" }); - const event = (i) => - `data: ${JSON.stringify({ - type: "message.part.delta", - properties: { - sessionID: i % 2 ? "ses_foreign" : ref.sessionId, - messageID: "msg", - partID: "part", - field: "text", - delta: JSON.stringify({ i, sentAt: performance.now() }), - }, - })}\n\n`; - if (!workload.intervalMs) { - res.end(Array.from({ length: workload.count }, (_, i) => event(i)).join("")); - return; - } - let index = 0; - const timer = setInterval(() => { - res.write(event(index++)); - if (index === workload.count) { - clearInterval(timer); - res.end(); - } - }, workload.intervalMs); - res.on("close", () => clearInterval(timer)); -}); -server.listen(0, "127.0.0.1"); -await NodeEvents.once(server, "listening"); -const baseUrl = `http://127.0.0.1:${server.address().port}`; -const clients = { - baseline: await Effect.runPromise(baseline.make({ ...ref, baseUrl })), - current: await Effect.runPromise(make({ ...ref, baseUrl })), -}; -const sdk = createKiloClient({ baseUrl, directory: ref.directory, throwOnError: true }); -const results = []; -try { - for (const scenario of ["burst", "paced"]) { - workload = - scenario === "burst" ? { count: 10000, intervalMs: 0 } : { count: 200, intervalMs: 5 }; - for (let round = 0; round < 8; round++) { - const order = round % 2 ? ["current", "baseline", "sdk"] : ["sdk", "baseline", "current"]; - for (const kind of order) { - global.gc(); - const heap = process.memoryUsage().heapUsed; - const cpu = process.cpuUsage(); - const began = performance.now(); - const indices = []; - const latency = []; - const consume = (event) => { - const delta = JSON.parse(event.properties.delta); - indices.push(delta.i); - latency.push(performance.now() - delta.sentAt); - }; - if (kind === "sdk") { - const subscription = await sdk.event.subscribe(undefined, { sseMaxRetryAttempts: 0 }); - for await (const event of subscription.stream) - if (event.properties.sessionID === ref.sessionId) consume(event); - } else { - const failure = await Effect.runPromise( - clients[kind].events(ref).pipe( - Stream.runForEach((event) => Effect.sync(() => consume(event))), - Effect.scoped, - Effect.flip, - ), - ); - NodeAssert.equal(failure.reason, "request_failed"); // EOF is not task completion. - } - const wallMs = performance.now() - began; - const used = process.cpuUsage(cpu); - NodeAssert.deepEqual( - indices, - Array.from({ length: workload.count / 2 }, (_, i) => i * 2), - ); - latency.sort((a, b) => a - b); - results.push({ - scenario, - round, - kind, - retained: indices.length, - wallMs, - cpuMs: (used.user + used.system) / 1000, - heapDelta: process.memoryUsage().heapUsed - heap, - latencyP50Ms: latency[Math.floor(latency.length * 0.5)], - latencyP95Ms: latency[Math.floor(latency.length * 0.95)], - }); - } - } - } - console.log( - JSON.stringify( - { - node: process.version, - cpu: NodeOS.cpus()[0]?.model, - baselinePath: process.argv[2], - buffering: "HTTP/SSE to serial consumer; no added queue", - results, - }, - null, - 2, - ), - ); -} finally { - server.closeAllConnections(); - await new Promise((resolve) => server.close(resolve)); -} diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts index c24883f0292a..51e4838bd15e 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts @@ -283,6 +283,7 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> instanceId: continuationKey, binaryPath: binary!, profileDirectory: path.join(root, "profile"), + processStateDirectory: path.join(root, "state"), environment: { PATH: process.env.PATH, HOME: root, diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts index b812f79fcbc3..568db8452862 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts @@ -183,6 +183,137 @@ const permissions = (policy: Adapter.ProviderAdapterV2RuntimePolicy) => { return effective; }; +type KiloInteraction = + | { readonly id: string; readonly permission: string; readonly patterns: ReadonlyArray } + | { + readonly id: string; + readonly questions: ReadonlyArray<{ + readonly header: string; + readonly question: string; + readonly options: ReadonlyArray<{ readonly label: string; readonly description: string }>; + readonly multiple?: boolean | undefined; + readonly custom?: boolean | undefined; + }>; + }; + +/** The runtime request, node and turn item that present a native Kilo permission or question. */ +export const kiloInteraction = (input: { + readonly native: KiloInteraction; + readonly requestId: RuntimeRequestId; + readonly nodeId: OrchestrationV2ExecutionNode["id"]; + readonly turnItemId: OrchestrationV2TurnItem["id"]; + readonly nativeRef: NonNullable; + readonly threadId: OrchestrationV2ExecutionNode["threadId"]; + readonly runId: OrchestrationV2ExecutionNode["runId"]; + readonly rootNodeId: OrchestrationV2ExecutionNode["id"]; + readonly providerThreadId: OrchestrationV2ProviderThread["id"]; + readonly providerTurnId: OrchestrationV2ProviderTurn["id"]; + readonly providerSessionId: Adapter.ProviderAdapterV2SessionRuntime["providerSessionId"]; + readonly ordinal: number; + readonly at: DateTime.Utc; +}) => { + const { native, at } = input; + const question = "questions" in native; + const kind = question + ? "user_input" + : /edit|write|patch/.test(native.permission) + ? "file-change" + : /read|glob|grep/.test(native.permission) + ? "file-read" + : "command"; + const runtime: OrchestrationV2RuntimeRequest = { + id: input.requestId, + nodeId: input.nodeId, + providerTurnId: input.providerTurnId, + nativeRequestRef: input.nativeRef, + kind, + status: "pending", + responseCapability: { type: "live", providerSessionId: input.providerSessionId }, + createdAt: at, + resolvedAt: null, + }; + const node: OrchestrationV2ExecutionNode = { + id: input.nodeId, + threadId: input.threadId, + runId: input.runId, + parentNodeId: input.rootNodeId, + rootNodeId: input.rootNodeId, + kind: question ? "user_input_request" : "approval_request", + status: "waiting", + countsForRun: false, + providerThreadId: input.providerThreadId, + providerTurnId: input.providerTurnId, + nativeItemRef: input.nativeRef, + runtimeRequestId: input.requestId, + checkpointScopeId: null, + startedAt: at, + completedAt: null, + }; + const base = { + id: input.turnItemId, + threadId: input.threadId, + runId: input.runId, + nodeId: input.nodeId, + providerThreadId: input.providerThreadId, + providerTurnId: input.providerTurnId, + nativeItemRef: input.nativeRef, + parentItemId: null, + ordinal: input.ordinal, + status: "waiting" as const, + startedAt: at, + completedAt: null, + updatedAt: at, + requestId: input.requestId, + }; + const turnItem: OrchestrationV2TurnItem = question + ? { + ...base, + type: "user_input_request", + title: "Kilo question", + questions: native.questions.map((q, index) => ({ + id: String(index), + header: q.header, + question: q.question, + options: q.options, + multiSelect: q.multiple ?? false, + allowCustomAnswer: q.custom ?? true, + })), + } + : { + ...base, + type: "approval_request", + title: native.permission, + requestKind: kind === "user_input" ? "command" : kind, + prompt: native.patterns.join("\n"), + }; + return { runtime, node, turnItem }; +}; + +/** Native answers in question order, or undefined when any question is unanswered. */ +export const kiloQuestionAnswers = ( + questions: ReadonlyArray, + answers: NonNullable, +) => { + const native = questions.map((_, index) => { + const answer = answers[String(index)]; + return typeof answer === "string" + ? [answer] + : Array.isArray(answer) && answer.every((value) => typeof value === "string") + ? answer + : []; + }); + return native.some((answer) => answer.length === 0) ? undefined : native; +}; + +export const kiloPermissionReply = ( + decision: NonNullable, +) => + decision === "accept" + ? ("once" as const) + : decision === "acceptForSession" || decision === "acceptAlways" + ? ("always" as const) + : ("reject" as const); + export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { readonly instanceId: ProviderInstanceId; readonly continuationKey: string; @@ -831,95 +962,36 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { native: PermissionRequest | QuestionRequest, ) { const running = active; - if (!running || !thread || requests.has(RuntimeRequestId.make(itemKey(native.id)))) return; - const at = yield* DateTime.now; const requestId = RuntimeRequestId.make(itemKey(native.id)); - const nodeId = ids.derive.approvalNode({ requestId }); - const question = "questions" in native; - const kind = question - ? "user_input" - : /edit|write|patch/.test(native.permission) - ? "file-change" - : /read|glob|grep/.test(native.permission) - ? "file-read" - : "command"; - const runtime: OrchestrationV2RuntimeRequest = { - id: requestId, - nodeId, + if (!running || !thread || requests.has(requestId)) return; + const interaction = kiloInteraction({ + native, + requestId, + nodeId: ids.derive.approvalNode({ requestId }), + turnItemId: ids.derive.approvalTurnItem({ requestId }), + nativeRef: nativeRef(native.id), + threadId: running.input.threadId, + runId: running.input.runId, + rootNodeId: running.input.rootNodeId, + providerThreadId: thread.id, providerTurnId: running.turn.id, - nativeRequestRef: nativeRef(native.id), - kind, - status: "pending", - responseCapability: { type: "live", providerSessionId: input.providerSessionId }, - createdAt: at, - resolvedAt: null, - }; - requests.set(requestId, { runtime, native }); - yield* emit({ - type: "node.updated", - driver: KILO_PROVIDER, - node: { - id: nodeId, - threadId: running.input.threadId, - runId: running.input.runId, - parentNodeId: running.input.rootNodeId, - rootNodeId: running.input.rootNodeId, - kind: question ? "user_input_request" : "approval_request", - status: "waiting", - countsForRun: false, - providerThreadId: thread.id, - providerTurnId: running.turn.id, - nativeItemRef: nativeRef(native.id), - runtimeRequestId: requestId, - checkpointScopeId: null, - startedAt: at, - completedAt: null, - }, + providerSessionId: input.providerSessionId, + ordinal: ordinal(native.id), + at: yield* DateTime.now, }); + requests.set(requestId, { runtime: interaction.runtime, native }); + yield* emit({ type: "node.updated", driver: KILO_PROVIDER, node: interaction.node }); yield* emit({ type: "runtime_request.updated", driver: KILO_PROVIDER, threadId: running.input.threadId, - runtimeRequest: runtime, + runtimeRequest: interaction.runtime, + }); + yield* emit({ + type: "turn_item.updated", + driver: KILO_PROVIDER, + turnItem: interaction.turnItem, }); - const base = { - id: ids.derive.approvalTurnItem({ requestId }), - threadId: running.input.threadId, - runId: running.input.runId, - nodeId, - providerThreadId: thread.id, - providerTurnId: running.turn.id, - nativeItemRef: nativeRef(native.id), - parentItemId: null, - ordinal: ordinal(native.id), - status: "waiting" as const, - startedAt: at, - completedAt: null, - updatedAt: at, - requestId, - }; - const turnItem: OrchestrationV2TurnItem = question - ? { - ...base, - type: "user_input_request", - title: "Kilo question", - questions: native.questions.map((q, index) => ({ - id: String(index), - header: q.header, - question: q.question, - options: q.options, - multiSelect: q.multiple ?? false, - allowCustomAnswer: q.custom ?? true, - })), - } - : { - ...base, - type: "approval_request", - title: native.permission, - requestKind: kind === "user_input" ? "command" : kind, - prompt: native.patterns.join("\n"), - }; - yield* emit({ type: "turn_item.updated", driver: KILO_PROVIDER, turnItem }); }); const handle = Effect.fn("KiloAdapterV2.event")(function* (event: Event) { const eventSession = @@ -1311,6 +1383,35 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { ); return yield* bind(native, saved.appThreadId, saved); }); + // Native revert changes files, so rewind and fork copy only the conversation before + // `next`; T3 owns filesystem rewind. Aliases map original message IDs to the copies. + const forkBefore = Effect.fn("KiloAdapterV2.forkBefore")(function* ( + native: KiloSessionRef, + history: Effect.Success>, + next: string | undefined, + ) { + const fork = yield* wire(client.fork(native, next)); + const retained = next + ? history.slice( + 0, + history.findIndex((m) => m.info.id === next), + ) + : history; + const copied = yield* wire(client.history(fork)); + if ( + copied.length !== retained.length || + copied.some((m, i) => m.info.role !== retained[i]?.info.role) + ) + return yield* error("Kilo fork returned an unexpected conversation boundary"); + const aliases = { ...thread?.nativeMetadata?.messageAliases }; + for (const [i, entry] of retained.entries()) { + const replacement = copied[i]!.info.id; + for (const [old, currentId] of Object.entries(aliases)) + if (currentId === entry.info.id) aliases[old] = replacement; + aliases[entry.info.id] = replacement; + } + return { fork, aliases }; + }); const runtime: Adapter.ProviderAdapterV2SessionRuntime = { instanceId: options.instanceId, driver: KILO_PROVIDER, @@ -1513,16 +1614,8 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { // The client rechecks both the session owner and pending request before replying. if ("questions" in pending.native) { if (!request.answers) return yield* error("Kilo question requires answers"); - const answers = pending.native.questions.map((_, index) => { - const answer = request.answers?.[String(index)]; - return typeof answer === "string" - ? [answer] - : Array.isArray(answer) && answer.every((value) => typeof value === "string") - ? answer - : []; - }); - if (answers.some((answer) => answer.length === 0)) - return yield* error("Each Kilo question requires a text answer"); + const answers = kiloQuestionAnswers(pending.native.questions, request.answers); + if (!answers) return yield* error("Each Kilo question requires a text answer"); yield* wire(client.replyQuestion(native, pending.native.id, answers)); } else { if (!request.decision) return yield* error("Kilo approval requires a decision"); @@ -1530,12 +1623,7 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { client.replyPermission( native, pending.native.id, - request.decision === "accept" - ? "once" - : request.decision === "acceptForSession" || - request.decision === "acceptAlways" - ? "always" - : "reject", + kiloPermissionReply(request.decision), ), ); } @@ -1599,25 +1687,7 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { const next = history.slice(targetIndex + 1).find((m) => m.info.role === "user") ?.info.id; if (!next) return yield* snapshot(); - // Native revert changes files. Fork only the conversation; T3 owns filesystem rewind. - const fork = yield* wire(client.fork(native, next)); - const retained = history.slice( - 0, - history.findIndex((m) => m.info.id === next), - ); - const copied = yield* wire(client.history(fork)); - if ( - copied.length !== retained.length || - copied.some((m, i) => m.info.role !== retained[i]?.info.role) - ) - return yield* error("Kilo fork returned an unexpected conversation boundary"); - const aliases = { ...thread?.nativeMetadata?.messageAliases }; - for (const [i, entry] of retained.entries()) { - const replacement = copied[i]!.info.id; - for (const [old, currentId] of Object.entries(aliases)) - if (currentId === entry.info.id) aliases[old] = replacement; - aliases[entry.info.id] = replacement; - } + const { fork, aliases } = yield* forkBefore(native, history, next); if (eventFiber) yield* Fiber.interrupt(eventFiber); subscribed = false; ref = fork; @@ -1664,25 +1734,7 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { const next = boundary ? history.slice(boundaryIndex + 1).find((m) => m.info.role === "user")?.info.id : undefined; - const fork = yield* wire(client.fork(native, next)); - const retained = next - ? history.slice( - 0, - history.findIndex((m) => m.info.id === next), - ) - : history; - const copied = yield* wire(client.history(fork)); - if ( - copied.length !== retained.length || - copied.some((m, i) => m.info.role !== retained[i]?.info.role) - ) - return yield* error("Kilo fork returned an unexpected conversation boundary"); - const aliases = { ...thread?.nativeMetadata?.messageAliases }; - for (const [i, entry] of retained.entries()) { - for (const [old, currentId] of Object.entries(aliases)) - if (currentId === entry.info.id) aliases[old] = copied[i]!.info.id; - aliases[entry.info.id] = copied[i]!.info.id; - } + const { fork, aliases } = yield* forkBefore(native, history, next); if (!thread) return yield* error("Kilo thread is not loaded"); return { ...thread, diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.live.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.live.test.ts deleted file mode 100644 index 61c015ac9b1b..000000000000 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.live.test.ts +++ /dev/null @@ -1,459 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off - opt-in live integration with loopback inference for local Kilo. -import * as NodeHttp from "node:http"; -import * as NodeEvents from "node:events"; -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; -import { describe } from "vite-plus/test"; -import { - OrchestrationV2ThreadProjection, - RunId, - RunAttemptId, - NodeId, - CommandId, - MessageId, - ProjectId, - ProviderInstanceId, - ThreadId, -} from "@t3tools/contracts"; -import * as Deferred from "effect/Deferred"; -import * as Clock from "effect/Clock"; -import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Layer from "effect/Layer"; -import * as Schema from "effect/Schema"; -import * as Stream from "effect/Stream"; -import * as Account from "../../provider/kilo/KiloCloudAccount.ts"; -import * as Cloud from "../../provider/kilo/KiloCloudWebClient.ts"; -import * as Journal from "../../provider/kilo/KiloCloudJournal.ts"; -import * as IdAllocator from "../IdAllocator.ts"; -import * as Orchestrator from "../Orchestrator.ts"; -import * as EffectWorker from "../EffectWorker.ts"; -import * as Registry from "../ProviderAdapterRegistry.ts"; -import { makeOrchestratorV2ReplayLayerWithRegistry } from "../testkit/ProviderReplayHarness.ts"; -import * as CloudAdapter from "./KiloCloudAdapterV2.ts"; -import type * as Adapter from "../ProviderAdapter.ts"; -import * as LocalAdapter from "./KiloAdapterV2.ts"; -import * as LocalRuntime from "../../provider/kilo/KiloRuntime.ts"; - -// Explicit paid opt-in, never enabled by CI. A durable one-shot directory prevents -// an accidental rerun from admitting a duplicate sandbox after an uncertain result. -const profile = process.env.KILO_CLOUD_TEST_PROFILE; -const evidence = process.env.KILO_CLOUD_PAID_EVIDENCE; -const repository = process.env.KILO_CLOUD_TEST_REPOSITORY; -const enabled = - process.env.KILO_CLOUD_ALLOW_PAID_TEST === "yes" && - process.env.KILO_CLOUD_ALLOW_FULL_ACCESS_READ_ONLY_TEST === "yes" && - process.env.KILO_BIN && - profile && - evidence && - repository; -const decodeProjection = Schema.decodeUnknownEffect( - Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadProjection)), -); -const json = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); -describe.skipIf(!enabled)("paid Kilo Cloud integration", () => { - it.live( - "reads a synthetic repository and follows up through Orchestrator without local uploads", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - yield* fs.makeDirectory(evidence!); // EEXIST deliberately prevents paid retries. - const account = yield* Account.make(profile!); - const credentials = yield* account.load; - const client = Cloud.make({ ...credentials, credentials: account.load }); - const journal = yield* Journal.make(`${evidence}/journal`); - const instanceId = ProviderInstanceId.make("kilo-cloud-paid-test"); - const threadId = ThreadId.make("kilo-cloud-paid-test"); - const modelSelection = { - instanceId, - model: "deepseek/deepseek-v4.1-flash", - options: [{ id: "variant", value: "low" }], - }; - const adapter = yield* CloudAdapter.make({ - instanceId, - continuationKey: "kilo-cloud-paid-test", - accountId: credentials.accountId, - repository: repository!, - branch: "main", - client, - journal, - }); - const localRoot = `${evidence}/local-workspace`; - yield* fs.makeDirectory(localRoot); - yield* fs.writeFileString(`${localRoot}/README.md`, "LOCAL_ONLY_SENTINEL_DO_NOT_UPLOAD"); - const inference = yield* Effect.acquireRelease( - Effect.promise(async () => { - const server = NodeHttp.createServer((request, response) => { - request.resume(); - request.on("end", () => { - response.writeHead(200, { "content-type": "text/event-stream" }); - for (const delta of [{ content: "LOCAL_ONLY_SENTINEL_DO_NOT_UPLOAD" }, {}]) - response.write( - `data: ${JSON.stringify({ id: "local", object: "chat.completion.chunk", created: 0, model: "test", choices: [{ index: 0, delta, finish_reason: Object.keys(delta).length ? null : "stop" }] })}\n\n`, - ); - response.end("data: [DONE]\n\n"); - }); - }); - server.listen(0, "127.0.0.1"); - await NodeEvents.EventEmitter.once(server, "listening"); - const address = server.address(); - if (!address || typeof address === "string") - throw new Error("Missing local fixture address"); - return { server, url: `http://127.0.0.1:${address.port}/v1` }; - }), - ({ server }) => - Effect.promise( - () => - new Promise((resolve) => { - server.closeAllConnections(); - server.close(() => resolve()); - }), - ), - ); - const localInstance = ProviderInstanceId.make("kilo-local-parallel"); - const localThreadId = ThreadId.make("kilo-local-parallel"); - const localModel = { instanceId: localInstance, model: "fixture/test", options: [] }; - const runtime = yield* LocalRuntime.make({ - instanceId: "parallel-local", - binaryPath: process.env.KILO_BIN!, - profileDirectory: `${evidence}/local-profile`, - environment: { - PATH: process.env.PATH, - HTTP_PROXY: process.env.HTTP_PROXY, - HTTPS_PROXY: process.env.HTTPS_PROXY, - NO_PROXY: process.env.NO_PROXY, - NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS, - KILO_DISABLE_AUTOUPDATE: "1", - KILO_DISABLE_MODELS_FETCH: "1", - KILO_DISABLE_DEFAULT_PLUGINS: "1", - KILO_DISABLE_EXTERNAL_SKILLS: "1", - KILO_DISABLE_PROJECT_CONFIG: "1", - KILO_CONFIG_CONTENT: json({ - model: "fixture/test", - small_model: "fixture/test", - plugin: [], - enabled_providers: ["fixture"], - provider: { - fixture: { - npm: "@ai-sdk/openai-compatible", - name: "Local", - options: { baseURL: inference.url }, - models: { test: { name: "Local", limit: { context: 10000, output: 1000 } } }, - }, - }, - }), - }, - }); - const localAdapter = yield* LocalAdapter.make({ - instanceId: localInstance, - continuationKey: "parallel-local", - cwd: localRoot, - runtime, - attachmentsDir: `${evidence}/attachments`, - }); - yield* Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const terminals = [yield* Deferred.make(), yield* Deferred.make()]; - const seen = new Set(); - const localDone = yield* Deferred.make(); - const timing: Array<{ type: string; threadId: string; status: string; at: number }> = []; - yield* orchestrator.streamStoredEvents.pipe( - Stream.runForEach(({ event }) => - Effect.gen(function* () { - if (event.type === "run.updated") - timing.push({ - type: event.type, - threadId: event.threadId, - status: event.payload.status, - at: yield* Clock.currentTimeMillis, - }); - if ( - event.type === "run.updated" && - event.threadId === localThreadId && - ["completed", "failed", "interrupted"].includes(event.payload.status) - ) - yield* Deferred.succeed(localDone, undefined); - if ( - event.threadId === threadId && - event.type === "run.updated" && - ["completed", "failed", "interrupted"].includes(event.payload.status) && - !seen.has(event.payload.id) - ) { - seen.add(event.payload.id); - const terminal = terminals[seen.size - 1]; - if (terminal) yield* Deferred.succeed(terminal, undefined); - } - }), - ), - Effect.forkScoped, - ); - yield* orchestrator.dispatch({ - type: "thread.create", - commandId: CommandId.make("cloud-live-create"), - createdBy: "user", - creationSource: "web", - threadId, - projectId: ProjectId.make("synthetic"), - title: "Read-only cloud integration", - modelSelection, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: `${evidence}/LOCAL-FILES-MUST-NOT-BE-READ`, - }); - const prompts = [ - "Read only README.md and list the top-level file names in this synthetic repository. Reply with a brief summary. Do not execute shell commands, edit files, commit, open a PR, access the network, or start subagents.", - "Using only the context already read, repeat one top-level file name. Do not use tools, modify files, commit, or open a PR. Keep the answer to one line.", - ]; - for (let index = 0; index < prompts.length; index++) { - yield* orchestrator.dispatch({ - type: "message.dispatch", - commandId: CommandId.make(`cloud-live-send-${index}`), - createdBy: "user", - creationSource: "web", - threadId, - messageId: MessageId.make(`cloud-live-user-${index}`), - text: prompts[index]!, - attachments: [], - modelSelection, - dispatchMode: { type: "start_immediately" }, - }); - if (index === 0) { - yield* orchestrator.dispatch({ - type: "thread.create", - commandId: CommandId.make("local-create"), - createdBy: "user", - creationSource: "web", - threadId: localThreadId, - projectId: ProjectId.make("local"), - title: "Parallel local isolation", - modelSelection: localModel, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: localRoot, - }); - yield* orchestrator.dispatch({ - type: "message.dispatch", - commandId: CommandId.make("local-send"), - createdBy: "user", - creationSource: "web", - threadId: localThreadId, - messageId: MessageId.make("local-user"), - text: "Say local hello", - attachments: [], - modelSelection: localModel, - dispatchMode: { type: "start_immediately" }, - }); - } - yield* (yield* EffectWorker.OrchestrationEffectWorkerV2).drain(); - yield* Deferred.await(terminals[index]!); - const projection = yield* orchestrator.getThreadProjection(threadId); - yield* fs.writeFileString(`${evidence}/turn-${index}.json`, json(projection)); - assert.equal(projection.runs.at(-1)?.status, "completed"); - assert.equal(projection.checkpoints.length, 0); - assert.isFalse( - projection.messages.some((message) => message.text.includes("LOCAL_ONLY_SENTINEL")), - ); - assert.isFalse(yield* fs.exists(`${evidence}/LOCAL-FILES-MUST-NOT-BE-READ`)); - assert.isTrue( - projection.messages.some( - (message) => message.role === "assistant" && message.text.length > 0, - ), - ); - } - yield* Deferred.await(localDone); - const localProjection = yield* orchestrator.getThreadProjection(localThreadId); - yield* fs.writeFileString( - `${evidence}/parallel-local.json`, - json({ projection: localProjection, timing }), - ); - assert.equal(localProjection.runs.at(-1)?.status, "completed"); - assert.isTrue( - localProjection.messages.some((message) => - message.text.includes("LOCAL_ONLY_SENTINEL"), - ), - ); - assert.isFalse( - localProjection.providerThreads.some((thread) => thread.driver === "kilo-cloud"), - ); - }).pipe( - Effect.provide( - makeOrchestratorV2ReplayLayerWithRegistry( - { - name: "kilo-cloud-paid", - }, - Registry.makeLayer([adapter, localAdapter]), - ), - ), - ); - const entries = yield* journal.read; - const binding = entries.at(-1)?.binding; - if (!binding) - return yield* Effect.die( - new Error("No cloud binding; inspect durable admission before any retry."), - ); - yield* fs.writeFileString( - `${evidence}/lifecycle-after.json`, - json({ - binding, - sandbox: yield* client.sandbox(binding), - billing: yield* client.billing(binding), - }), - ); - }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), - 240_000, - ); -}); - -const recovery = process.env.KILO_CLOUD_RECOVER_EVIDENCE; -describe.skipIf(!profile || !recovery)("read-only Kilo Cloud recovery", () => { - it.live( - "reconciles a saved admission through the adapter without resubmitting", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const account = yield* Account.make(profile!); - const credentials = yield* account.load; - const journal = yield* Journal.make(`${recovery}/journal`); - const intent = (yield* journal.read).at(-1); - if (!intent || !intent.binding) - return yield* Effect.die(new Error("Missing saved cloud admission")); - assert.equal(intent.accountId, credentials.accountId); - const client = Cloud.make({ ...credentials, credentials: account.load }); - const adapter = yield* CloudAdapter.make({ - instanceId: intent.providerThread.providerInstanceId, - continuationKey: intent.providerThread.nativeMetadata!.continuationKey!, - accountId: credentials.accountId, - repository: intent.repository, - branch: intent.branch, - client, - journal, - }); - const session = yield* adapter.openSession({ - threadId: intent.providerThread.appThreadId!, - providerSessionId: intent.providerThread.providerSessionId!, - modelSelection: { - instanceId: intent.providerThread.providerInstanceId, - model: "deepseek/deepseek-v4.1-flash", - }, - runtimePolicy: { - runtimeMode: "approval-required", - interactionMode: "default", - cwd: null, - }, - }); - const thread = yield* session.resumeThread({ providerThread: intent.providerThread }); - const snapshot = yield* session.readThreadSnapshot({ providerThread: thread }); - const saved = (yield* journal.read).at(-1)!; - yield* fs.writeFileString( - `${recovery}/recovered.json`, - json({ - state: saved.state, - operationKey: saved.operationKey, - messageId: saved.messageId, - snapshot, - sandbox: yield* client.sandbox(intent.binding), - billing: yield* client.billing(intent.binding), - }), - ); - assert.isTrue(["completed", "failed", "interrupted"].includes(saved.state)); - assert.equal(saved.operationKey, intent.operationKey); - }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), - 60_000, - ); -}); - -const controlEvidence = process.env.KILO_CLOUD_CONTROL_EVIDENCE; -describe.skipIf( - !profile || !controlEvidence || process.env.KILO_CLOUD_ALLOW_FULL_ACCESS_READ_ONLY_TEST !== "yes", -)("paid existing-session interrupt", () => { - it.live( - "restores native history and confirms remote interruption in the same cloud worktree", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - yield* fs.makeDirectory(`${controlEvidence}/interrupt-admission`); - const projection = yield* decodeProjection( - yield* fs.readFileString(`${controlEvidence}/turn-1.json`), - ); - const journal = yield* Journal.make(`${controlEvidence}/journal`); - const prior = (yield* journal.read).at(-1)!; - assert.equal(prior.state, "completed"); - const account = yield* Account.make(profile!); - const credentials = yield* account.load; - assert.equal(credentials.accountId, prior.accountId); - const client = Cloud.make({ ...credentials, credentials: account.load }); - const adapter = yield* CloudAdapter.make({ - instanceId: prior.providerThread.providerInstanceId, - continuationKey: prior.providerThread.nativeMetadata!.continuationKey!, - accountId: credentials.accountId, - repository: prior.repository, - branch: prior.branch, - client, - journal, - }); - const runtimePolicy = { - runtimeMode: "full-access" as const, - interactionMode: "default" as const, - cwd: null, - }; - const session = yield* adapter.openSession({ - threadId: projection.thread.id, - providerSessionId: prior.providerThread.providerSessionId!, - modelSelection: projection.thread.modelSelection, - runtimePolicy, - }); - const thread = yield* session.resumeThread({ providerThread: prior.providerThread }); - const restored = yield* session.readThreadSnapshot({ providerThread: thread }); - assert.isTrue(restored.messages.some((message) => message.text === "fixture.py")); - const terminal = yield* Deferred.make(); - yield* session.events.pipe( - Stream.runForEach((event) => - event.type === "turn.terminal" - ? Deferred.succeed(terminal, event).pipe(Effect.asVoid) - : Effect.void, - ), - Effect.forkScoped, - ); - yield* session.startTurn({ - appThread: projection.thread, - threadId: projection.thread.id, - providerThread: thread, - runId: RunId.make("cloud-stop-test"), - runOrdinal: 3, - providerTurnOrdinal: 3, - attemptId: RunAttemptId.make("cloud-stop-test"), - rootNodeId: NodeId.make("cloud-stop-test"), - message: { - messageId: MessageId.make("cloud-stop-test"), - text: "Read-only interruption check: count from one to 200, one number per line. Do not use tools, modify files, commit, open a PR, or start subagents.", - attachments: [], - createdBy: "user", - creationSource: "web", - }, - modelSelection: projection.thread.modelSelection, - runtimePolicy, - }); - const admitted = (yield* journal.read).at(-1)!; - assert.equal(admitted.binding?.worktreeId, prior.binding?.worktreeId); - yield* session.interruptTurn({ - providerThread: thread, - providerTurnId: admitted.providerTurn.id, - }); - const event = yield* Deferred.await(terminal); - const result = yield* client.result(admitted.binding!, admitted.messageId); - yield* fs.writeFileString( - `${controlEvidence}/interrupt-result.json`, - json({ - event, - result, - sandbox: yield* client.sandbox(admitted.binding!), - billing: yield* client.billing(admitted.binding!), - }), - ); - assert.equal(result?.status, "interrupted"); - }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), - 90_000, - ); -}); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index c83a1cbb26bb..846e3339efe6 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -592,6 +592,7 @@ it.live( instanceId: "native-parallel", binaryPath: process.env.KILO_BIN, profileDirectory: `${directory}/native-profile`, + processStateDirectory: `${directory}/native-state`, environment: { PATH: process.env.PATH, HOME: directory, diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts index c33bb46da8e4..3d2f25fa72de 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -23,7 +23,7 @@ import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as Cloud from "../../provider/kilo/KiloCloudWebClient.ts"; -import { KiloCloudError } from "../../provider/kilo/KiloCloudClient.ts"; +import { KiloCloudError } from "../../provider/kilo/KiloCloudError.ts"; import * as Journal from "../../provider/kilo/KiloCloudJournal.ts"; import * as IdAllocator from "../IdAllocator.ts"; import * as Adapter from "../ProviderAdapter.ts"; @@ -32,6 +32,7 @@ import { makeProviderFailure } from "../ProviderFailure.ts"; import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { openCodePermissionRules } from "./OpenCodeAdapterV2.ts"; import { openCodeToolTurnItem } from "./OpenCodeToolItems.ts"; +import { kiloInteraction, kiloPermissionReply, kiloQuestionAnswers } from "./KiloAdapterV2.ts"; export const KILO_CLOUD_PROVIDER = ProviderDriverKind.make("kilo-cloud"); const capabilities: OrchestrationV2ProviderCapabilities = { @@ -557,106 +558,37 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { const intent = active; const correlation = intent?.providerThread.nativeMetadata?.turnCorrelations?.[intent.messageId]; - if ( - !intent || - !correlation || - !thread?.appThreadId || - requests.has(RuntimeRequestId.make(key(native.id))) - ) - return; - const running = { - turn: intent.providerTurn, - input: { - threadId: thread.appThreadId, - runId: correlation.runId, - rootNodeId: correlation.nodeId, - }, - }; - const at = yield* DateTime.now; const requestId = RuntimeRequestId.make(key(native.id)); - const nodeId = ids.derive.approvalNode({ requestId }); - const question = "questions" in native; - const kind = question - ? "user_input" - : /edit|write|patch/.test(native.permission) - ? "file-change" - : /read|glob|grep/.test(native.permission) - ? "file-read" - : "command"; - const runtime: OrchestrationV2RuntimeRequest = { - id: requestId, - nodeId, - providerTurnId: running.turn.id, - nativeRequestRef: nativeRef(native.id), - kind, - status: "pending", - responseCapability: { type: "live", providerSessionId: input.providerSessionId }, - createdAt: at, - resolvedAt: null, - }; - const node: OrchestrationV2ExecutionNode = { - id: nodeId, - threadId: running.input.threadId, - runId: running.input.runId, - parentNodeId: running.input.rootNodeId, - rootNodeId: running.input.rootNodeId, - kind: question ? "user_input_request" : "approval_request", - status: "waiting", - countsForRun: false, + if (!intent || !correlation || !thread?.appThreadId || requests.has(requestId)) return; + const interaction = kiloInteraction({ + native, + requestId, + nodeId: ids.derive.approvalNode({ requestId }), + turnItemId: ids.derive.approvalTurnItem({ requestId }), + nativeRef: nativeRef(native.id), + threadId: thread.appThreadId, + runId: correlation.runId, + rootNodeId: correlation.nodeId, providerThreadId: thread.id, - providerTurnId: running.turn.id, - nativeItemRef: nativeRef(native.id), - runtimeRequestId: requestId, - checkpointScopeId: null, - startedAt: at, - completedAt: null, - }; - yield* emit({ type: "node.updated", driver, node }); + providerTurnId: intent.providerTurn.id, + providerSessionId: input.providerSessionId, + ordinal: ordinal(native.id), + at: yield* DateTime.now, + }); + requests.set(requestId, { + runtime: interaction.runtime, + native, + node: interaction.node, + item: interaction.turnItem, + }); + yield* emit({ type: "node.updated", driver, node: interaction.node }); yield* emit({ type: "runtime_request.updated", - driver: driver, - threadId: running.input.threadId, - runtimeRequest: runtime, + driver, + threadId: thread.appThreadId, + runtimeRequest: interaction.runtime, }); - const base = { - id: ids.derive.approvalTurnItem({ requestId }), - threadId: running.input.threadId, - runId: running.input.runId, - nodeId, - providerThreadId: thread.id, - providerTurnId: running.turn.id, - nativeItemRef: nativeRef(native.id), - parentItemId: null, - ordinal: ordinal(native.id), - status: "waiting" as const, - startedAt: at, - completedAt: null, - updatedAt: at, - requestId, - }; - const turnItem: OrchestrationV2TurnItem = question - ? { - ...base, - type: "user_input_request", - title: "Kilo question", - questions: native.questions.map((q, index) => ({ - id: String(index), - header: q.header, - question: q.question, - options: q.options, - multiSelect: q.multiple ?? false, - allowCustomAnswer: q.custom ?? true, - })), - } - : { - ...base, - type: "approval_request", - title: native.permission, - requestKind: kind === "user_input" ? "command" : kind, - prompt: native.patterns.join("\n"), - }; - requests.set(requestId, { runtime, native, node, item: turnItem }); - yield* emit({ type: "turn_item.updated", driver: driver, turnItem }); + yield* emit({ type: "turn_item.updated", driver, turnItem: interaction.turnItem }); }); const refreshIntent = Effect.gen(function* () { if (!active) return; @@ -1711,16 +1643,11 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { return yield* error("The cloud interaction has already ended."); let reply; if ("questions" in pending.native) { - const answers = pending.native.questions.map((_, index) => { - const value = response.answers?.[String(index)]; - return typeof value === "string" - ? [value] - : Array.isArray(value) && value.every((answer) => typeof answer === "string") - ? value - : []; - }); - if (answers.some((answer) => answer.length === 0)) - return yield* error("Each cloud question requires an answer."); + const answers = kiloQuestionAnswers( + pending.native.questions, + response.answers ?? {}, + ); + if (!answers) return yield* error("Each cloud question requires an answer."); reply = options.client.replyQuestion(binding, pending.native.id, answers); } else { if (!response.decision) @@ -1728,12 +1655,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { reply = options.client.replyPermission( binding, pending.native.id, - response.decision === "accept" - ? "once" - : response.decision === "acceptForSession" || - response.decision === "acceptAlways" - ? "always" - : "reject", + kiloPermissionReply(response.decision), ); } yield* save({ diff --git a/apps/server/src/provider/OpenCodeServerLedger.test.ts b/apps/server/src/provider/OpenCodeServerLedger.test.ts index 77e091d753ba..99058ab8fe14 100644 --- a/apps/server/src/provider/OpenCodeServerLedger.test.ts +++ b/apps/server/src/provider/OpenCodeServerLedger.test.ts @@ -242,119 +242,3 @@ describe.skipIf(observedPlatforms.length === 0)("OpenCode server startup", () => }).pipe(Effect.provide(NodeServices.layer)), ); }); - -for (const cancel of [false, true]) - it.live.skipIf(hostPlatform !== "linux")( - `serializes boot and profile reapers through group cleanup with cancellation=${cancel}`, - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const stateDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-ledger-overlap-" }); - const group = yield* Effect.acquireRelease( - Effect.gen(function* () { - const ready = yield* Deferred.make(); - const exited = yield* Deferred.make(); - const outputClosed = yield* Deferred.make(); - const leader = NodeChildProcess.spawn( - process.execPath, - [ - "-e", - ` - const {spawn}=require('node:child_process'); - const child=spawn(process.execPath,['-e',"process.on('SIGTERM',()=>{});process.send(process.pid);setInterval(()=>{},1000)"],{stdio:['ignore',process.stdout,'ignore','ipc']}); - child.once('message',pid=>process.send(pid)); - setInterval(()=>{},1000); - `, - ], - { detached: true, stdio: ["ignore", "pipe", "ignore", "ipc"] }, - ); - leader.once("message", (pid) => - Deferred.doneUnsafe(ready, Effect.succeed(Number(pid))), - ); - leader.once("exit", () => Deferred.doneUnsafe(exited, Effect.void)); - leader.stdout!.once("end", () => Deferred.doneUnsafe(outputClosed, Effect.void)); - leader.stdout!.resume(); - return { - pid: leader.pid!, - ready: Deferred.await(ready), - exited: Deferred.await(exited), - outputClosed: Deferred.await(outputClosed), - }; - }), - (group) => killGroup(group.pid), - ); - const member = yield* group.ready; - yield* recordFromDeadServer(stateDir, group); - const boot = yield* OpenCodeServerLedger.make({ stateDir }); - const profile = yield* OpenCodeServerLedger.make({ stateDir }); - const complete = yield* Deferred.make(); - const first = yield* boot.reapOrphans.pipe( - Effect.ensuring(Deferred.succeed(complete, undefined)), - Effect.forkScoped, - ); - yield* group.exited; // TERM killed the recorded leader; its child ignores TERM. - const running = fs.readFileString(`/proc/${member}/stat`).pipe( - Effect.map((stat) => !stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")), - Effect.orElseSucceed(() => false), - ); - expect(yield* running).toBe(true); - if (cancel) yield* Fiber.interrupt(first).pipe(Effect.forkScoped); - yield* profile.reapOrphans; - expect(yield* Deferred.isDone(complete)).toBe(true); - expect(yield* running).toBe(false); - yield* group.outputClosed; - const outcome = yield* Fiber.await(first); - expect(Exit.isFailure(outcome)).toBe(cancel); - // Coordination is released after the scan, not cached forever. - const later = yield* spawnGroup(SERVE_ARGS); - yield* recordFromDeadServer(stateDir, later); - yield* profile.reapOrphans; - yield* later.exited; - expect(groupExists(later.pid)).toBe(false); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - 20_000, - ); - -describe.skipIf(hostPlatform !== "linux")("interruptible discovery", () => { - it.live.each(["readDirectory", "readFileString", "remove"] as const)( - "cancels stalled ledger %s and releases the handoff gate", - (operation) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const stateDir = yield* fs.makeTempDirectoryScoped(); - const group = yield* spawnGroup(SERVE_ARGS); - yield* recordFromDeadServer(stateDir, group); - const entered = yield* Deferred.make(); - const release = yield* Deferred.make(); - const stall = (kind: typeof operation, path: string) => - kind === operation && path.includes("opencode-servers") - ? Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release))) - : Effect.void; - const stalled: FileSystem.FileSystem = { - ...fs, - readDirectory: (...args) => - stall("readDirectory", args[0]).pipe(Effect.andThen(fs.readDirectory(...args))), - readFileString: (...args) => - stall("readFileString", args[0]).pipe(Effect.andThen(fs.readFileString(...args))), - remove: (...args) => stall("remove", args[0]).pipe(Effect.andThen(fs.remove(...args))), - }; - const blocked = yield* OpenCodeServerLedger.make({ stateDir }).pipe( - Effect.provideService(FileSystem.FileSystem, stalled), - ); - const second = yield* OpenCodeServerLedger.make({ stateDir }); - const running = yield* blocked.reapOrphans.pipe(Effect.forkScoped); - yield* Deferred.await(entered); - const cancelled = yield* Fiber.interrupt(running).pipe( - Effect.timeout("1 second"), - Effect.exit, - Effect.ensuring(Deferred.succeed(release, undefined)), - ); - yield* Fiber.await(running); - expect(Exit.isSuccess(cancelled)).toBe(true); - yield* second.reapOrphans; - yield* group.exited; - expect(groupExists(group.pid)).toBe(false); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - 20_000, - ); -}); diff --git a/apps/server/src/provider/OpenCodeServerLedger.ts b/apps/server/src/provider/OpenCodeServerLedger.ts index 7f663eb3551e..654e4c0c2889 100644 --- a/apps/server/src/provider/OpenCodeServerLedger.ts +++ b/apps/server/src/provider/OpenCodeServerLedger.ts @@ -6,17 +6,11 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; -import * as Semaphore from "effect/Semaphore"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import * as ServerConfig from "../config.ts"; import { signalProcessGroup } from "../process/processGroup.ts"; -// Boot and provider runtimes create separate ledger instances in one T3 process. -// Serialize their scans, including the process-group grace period. A second scan -// must still run afterward to cover entries created since the first scan began. -const reapers = new Map(); - const ProcessIdentity = Schema.Struct({ pid: Schema.Int, startTime: Schema.String }); type ProcessIdentity = typeof ProcessIdentity.Type; @@ -286,14 +280,12 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { pid: entry.pgid, port: entry.port, }); - yield* Effect.gen(function* () { - signalGroup(entry.pgid, "SIGTERM"); - for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS && groupExists(entry.pgid); attempt++) { - yield* Effect.sleep(STOP_POLL_INTERVAL); - } - // The group never emptied, so its pgid cannot have been reused. - if (groupExists(entry.pgid)) signalGroup(entry.pgid, "SIGKILL"); - }).pipe(Effect.uninterruptible); + signalGroup(entry.pgid, "SIGTERM"); + for (let attempt = 0; attempt < STOP_POLL_ATTEMPTS && groupExists(entry.pgid); attempt++) { + yield* Effect.sleep(STOP_POLL_INTERVAL); + } + // The group never emptied, so its pgid cannot have been reused. + if (groupExists(entry.pgid)) signalGroup(entry.pgid, "SIGKILL"); }); const reapEntry = (entryPath: string) => @@ -311,7 +303,7 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { ); /** Stops recorded servers whose owning T3 server is gone and drops stale entries. */ - const reapOnce = Effect.gen(function* () { + const reapOrphans = Effect.gen(function* () { const names = yield* fs.readDirectory(directory).pipe(Effect.orElseSucceed(() => [])); yield* Effect.forEach( names.filter((name) => ENTRY_FILE.test(name)), @@ -320,29 +312,6 @@ export const make = Effect.fn("OpenCodeServerLedger.make")(function* (input: { ); }); - const reapOrphans = Effect.gen(function* () { - const resolved = path.resolve(input.stateDir); - const key = yield* fs.realPath(resolved).pipe(Effect.orElseSucceed(() => resolved)); - yield* Effect.acquireUseRelease( - Effect.sync(() => { - let entry = reapers.get(key); - if (!entry) { - entry = { gate: Semaphore.makeUnsafe(1), users: 0 }; - reapers.set(key, entry); - } - entry.users++; - return entry; - }), - // Discovery and filesystem cleanup remain interruptible. stopOrphan holds - // this permit through its bounded post-signal process-group cleanup. - (entry) => entry.gate.withPermit(reapOnce), - (entry) => - Effect.sync(() => { - if (--entry.users === 0) reapers.delete(key); - }), - ); - }); - return { track, reapOrphans }; }); diff --git a/apps/server/src/provider/kilo/KiloCloudAccount.ts b/apps/server/src/provider/kilo/KiloCloudAccount.ts index d346435f6204..0c07e62fa223 100644 --- a/apps/server/src/provider/kilo/KiloCloudAccount.ts +++ b/apps/server/src/provider/kilo/KiloCloudAccount.ts @@ -4,7 +4,7 @@ import * as Path from "effect/Path"; import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { KiloCloudError } from "./KiloCloudClient.ts"; +import { KiloCloudError } from "./KiloCloudError.ts"; const isKiloCloudError = Schema.is(KiloCloudError); diff --git a/apps/server/src/provider/kilo/KiloCloudClient.test.ts b/apps/server/src/provider/kilo/KiloCloudClient.test.ts deleted file mode 100644 index 6cbb2a8cd3ac..000000000000 --- a/apps/server/src/provider/kilo/KiloCloudClient.test.ts +++ /dev/null @@ -1,152 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off - exercises the customer HTTP contract over a real loopback socket. -import * as NodeHttp from "node:http"; -import * as NodeEvents from "node:events"; -import { afterEach, describe, expect, it } from "vite-plus/test"; -import * as Effect from "effect/Effect"; -import * as Redacted from "effect/Redacted"; -import * as Cloud from "./KiloCloudClient.ts"; - -const cleanups: Array<() => Promise> = []; -afterEach(async () => { - for (const cleanup of cleanups.splice(0)) await cleanup(); -}); -const sessionId = "agent_12345678-1234-1234-1234-123456789abc"; -const messageId = "msg_0123456789ab0123456789ABCD"; -const ref: Cloud.KiloCloudRef = { accountKey: "account-a", sessionId, messageId }; -const start = { - messageId, - prompt: "Contract fixture only", - repository: { type: "github" as const, repo: "fixture/project" }, - model: "fixture/model", - mode: "code", -}; -const run = Effect.runPromise; -async function client( - handler: (request: NodeHttp.IncomingMessage, response: NodeHttp.ServerResponse) => void, -) { - const server = NodeHttp.createServer(handler); - server.listen(0, "127.0.0.1"); - await NodeEvents.EventEmitter.once(server, "listening"); - const address = server.address(); - if (!address || typeof address === "string") throw new Error("Missing fixture address"); - cleanups.push(async () => { - server.closeAllConnections(); - await new Promise((resolve) => server.close(() => resolve())); - }); - return Cloud.make({ - accountKey: ref.accountKey, - apiKey: Redacted.make("fixture-customer-token"), - origin: `http://127.0.0.1:${address.port}`, - }); -} -function json(response: NodeHttp.ServerResponse, data: unknown) { - response.writeHead(200, { "content-type": "application/json" }); - response.end(JSON.stringify({ result: { data } })); -} -describe("Kilo customer Cloud Agent boundary", () => { - it("sends one bearer-authenticated admission and validates its caller-persisted message ID", async () => { - const requests: unknown[] = []; - const cloud = await client((request, response) => { - expect(request.headers.authorization).toBe("Bearer fixture-customer-token"); - expect(request.url).toBe("/trpc/start"); - let body = ""; - request.on("data", (chunk) => { - body += String(chunk); - }); - request.on("end", () => { - requests.push(JSON.parse(body)); - json(response, { - cloudAgentSessionId: sessionId, - kiloSessionId: "ses_remote", - messageId, - delivery: "queued", - }); - }); - }); - expect(await run(cloud.start(start))).toEqual(ref); - expect(requests).toEqual([ - { - message: { id: messageId, prompt: start.prompt }, - repository: start.repository, - agent: { model: start.model, mode: start.mode }, - options: { createdOnPlatform: "kilo-cli" }, - }, - ]); - }); - it("does not retry or declare stopped after a response is lost following admission", async () => { - let accepted = 0; - const cloud = await client((request, response) => { - request.resume(); - request.on("end", () => { - accepted++; - response.destroy(); - }); - }); - const failure = await run(cloud.start(start).pipe(Effect.flip)); - expect(failure.reason).toBe("admission_unknown"); - expect(failure.messageId).toBe(messageId); - expect(accepted).toBe(1); - }); - it.each([408, 409, 500, 503, 307])( - "never follows or retries mutation status %i", - async (status) => { - let requests = 0; - const cloud = await client((_request, response) => { - requests++; - response.writeHead(status, { location: "/other" }); - response.end(); - }); - expect((await run(cloud.start(start).pipe(Effect.flip))).reason).toBe("admission_unknown"); - expect(requests).toBe(1); - }, - ); - it("rejects a mismatched session on send and a mismatched message on result", async () => { - const cloud = await client((request, response) => - json( - response, - request.method === "POST" - ? { - cloudAgentSessionId: "agent_aaaaaaaa-1234-1234-1234-123456789abc", - messageId, - delivery: "started", - } - : { - cloudAgentSessionId: sessionId, - messageId: "msg_aaaaaaaaaaaa0123456789ABCD", - status: "completed", - createdAt: 1, - }, - ), - ); - expect((await run(cloud.send(ref, messageId, "test").pipe(Effect.flip))).reason).toBe( - "admission_unknown", - ); - expect((await run(cloud.result(ref).pipe(Effect.flip))).reason).toBe("wrong_owner"); - }); - it("cannot reuse another account's ref or issue an invented stop request", async () => { - let requests = 0; - const cloud = await client((_request, response) => { - requests++; - response.end(); - }); - expect( - (await run(cloud.result({ ...ref, accountKey: "account-b" }).pipe(Effect.flip))).reason, - ).toBe("wrong_owner"); - expect((await run(cloud.interrupt(ref).pipe(Effect.flip))).reason).toBe("unsupported"); - expect(requests).toBe(0); - }); - it("keeps billing unknown even when a task has a confirmed terminal result", async () => { - const cloud = await client((_request, response) => - json(response, { - cloudAgentSessionId: sessionId, - messageId, - status: "interrupted", - createdAt: 1, - terminalAt: 2, - }), - ); - const result = await run(cloud.result(ref)); - expect(result.status).toBe("interrupted"); - expect(result.billingStatus).toBe("unknown"); - }); -}); diff --git a/apps/server/src/provider/kilo/KiloCloudClient.ts b/apps/server/src/provider/kilo/KiloCloudClient.ts deleted file mode 100644 index 1c653a024223..000000000000 --- a/apps/server/src/provider/kilo/KiloCloudClient.ts +++ /dev/null @@ -1,218 +0,0 @@ -import * as Effect from "effect/Effect"; -import * as Redacted from "effect/Redacted"; -import * as Schema from "effect/Schema"; -import * as Stream from "effect/Stream"; -import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; - -/** Customer bearer endpoints used by the public Kilo 7.8.3 CLI, not the private cloud SDK. - * Reference: Kilo-Org/kilo packages/opencode/src/kilocode/cloud/{trpc,contracts}.ts. - * This boundary deliberately cannot report that billing stopped or invent a remote Stop route. - */ -export class KiloCloudError extends Schema.TaggedError()("KiloCloudError", { - operation: Schema.String, - reason: Schema.Literals([ - "rejected", - "not_found", - "admission_unknown", - "invalid_response", - "wrong_owner", - "unsupported", - "recovery_incomplete", - "recovery_limit", - ]), - recoveryCause: Schema.optional(Schema.String), - messageId: Schema.optional(Schema.String), -}) {} - -export const KiloCloudRef = Schema.Struct({ - accountKey: Schema.NonEmptyString, - sessionId: Schema.String.check(Schema.isPattern(/^agent_[0-9a-f-]{36}$/i)), - messageId: Schema.String.check(Schema.isPattern(/^msg_[0-9a-f]{12}[0-9A-Za-z]{14}$/)), -}); -export type KiloCloudRef = typeof KiloCloudRef.Type; -const Admission = Schema.Struct({ - cloudAgentSessionId: KiloCloudRef.fields.sessionId, - kiloSessionId: Schema.optional(Schema.String), - messageId: KiloCloudRef.fields.messageId, - delivery: Schema.NonEmptyString, -}); -const Result = Schema.Struct({ - cloudAgentSessionId: KiloCloudRef.fields.sessionId, - messageId: KiloCloudRef.fields.messageId, - status: Schema.Literals(["queued", "running", "completed", "failed", "interrupted"]), - createdAt: Schema.Number, - terminalAt: Schema.optional(Schema.Number), - assistant: Schema.optional( - Schema.Struct({ messageId: Schema.String, text: Schema.optional(Schema.String) }), - ), -}); -const Envelope = Schema.Struct({ result: Schema.Struct({ data: Schema.Unknown }) }); -const decodeEnvelope = Schema.decodeUnknownEffect(Schema.fromJsonString(Envelope)); -const decodeAdmission = Schema.decodeUnknownEffect(Admission); -const decodeResult = Schema.decodeUnknownEffect(Result); -const isCloudError = Schema.is(KiloCloudError); -const encode = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); - -export const make = (input: { - readonly accountKey: string; - readonly apiKey: Redacted.Redacted; - /** Only the production customer endpoint or a local contract-test server. */ - readonly origin?: string; -}) => { - const origin = input.origin ?? "https://cloud-agent-next.kilosessions.ai"; - const allowedOrigin = - origin === "https://cloud-agent-next.kilosessions.ai" || - /^http:\/\/127\.0\.0\.1:\d+$/.test(origin); - const own = (ref: KiloCloudRef) => - ref.accountKey === input.accountKey - ? Effect.void - : Effect.fail(new KiloCloudError({ operation: "ownership", reason: "wrong_owner" })); - const request = ( - operation: "start" | "send" | "getMessageResult", - body: unknown, - messageId: string, - ) => { - const mutation = operation !== "getMessageResult"; - const uncertain = () => - new KiloCloudError({ - operation, - reason: mutation ? "admission_unknown" : "invalid_response", - messageId, - }); - if (!allowedOrigin) return Effect.fail(new KiloCloudError({ operation, reason: "rejected" })); - return Effect.gen(function* () { - const client = yield* HttpClient.HttpClient; - const req = HttpClientRequest.make(mutation ? "POST" : "GET")( - `${origin}/trpc/${operation}${mutation ? "" : `?input=${encodeURIComponent(encode(body))}`}`, - { - headers: { - authorization: `Bearer ${Redacted.value(input.apiKey)}`, - "content-type": "application/json", - }, - }, - ); - const response = yield* client.execute( - mutation ? HttpClientRequest.bodyText(req, encode(body), "application/json") : req, - ); - if (response.status < 200 || response.status >= 300) - return yield* response.status >= 500 || response.status === 408 || response.status === 409 - ? uncertain() - : new KiloCloudError({ operation, reason: "rejected", messageId }); - const bytes = yield* response.stream.pipe( - Stream.runFoldEffect( - () => ({ size: 0, chunks: [] as Uint8Array[] }), - (acc, chunk) => - acc.size + chunk.byteLength > 1024 * 1024 - ? Effect.fail(uncertain()) - : Effect.succeed({ - size: acc.size + chunk.byteLength, - chunks: [...acc.chunks, chunk], - }), - ), - ); - return yield* decodeEnvelope(Buffer.concat(bytes.chunks).toString("utf8")).pipe( - Effect.map((envelope) => envelope.result.data), - ); - }).pipe( - Effect.scoped, - Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), - Effect.provide(FetchHttpClient.layer), - Effect.timeout("30 seconds"), - Effect.mapError((error) => (isCloudError(error) ? error : uncertain())), - ); - }; - - const admit = ( - operation: "start" | "send", - body: unknown, - messageId: string, - sessionId?: string, - ) => - request(operation, body, messageId).pipe( - Effect.flatMap((raw) => - decodeAdmission(raw).pipe( - Effect.mapError( - () => new KiloCloudError({ operation, reason: "admission_unknown", messageId }), - ), - ), - ), - Effect.flatMap((accepted) => - accepted.messageId !== messageId || - (sessionId !== undefined && accepted.cloudAgentSessionId !== sessionId) - ? Effect.fail(new KiloCloudError({ operation, reason: "admission_unknown", messageId })) - : Effect.succeed({ - accountKey: input.accountKey, - sessionId: accepted.cloudAgentSessionId, - messageId, - }), - ), - ); - return { - /** The caller must durably persist this message ID BEFORE submitting. An unknown start has - * no session ID to query through this customer API; operator reconciliation is required. - */ - start: (request: { - readonly messageId: string; - readonly prompt: string; - readonly repository: { - readonly type: "github"; - readonly repo: string; - readonly branch?: string; - }; - readonly model: string; - readonly mode: string; - }) => - admit( - "start", - { - message: { id: request.messageId, prompt: request.prompt }, - repository: request.repository, - agent: { model: request.model, mode: request.mode }, - options: { createdOnPlatform: "kilo-cli" }, - }, - request.messageId, - ), - send: (ref: KiloCloudRef, messageId: string, prompt: string) => - own(ref).pipe( - Effect.andThen( - admit( - "send", - { cloudAgentSessionId: ref.sessionId, message: { id: messageId, prompt } }, - messageId, - ref.sessionId, - ), - ), - ), - result: (ref: KiloCloudRef) => - own(ref).pipe( - Effect.andThen( - request( - "getMessageResult", - { cloudAgentSessionId: ref.sessionId, messageId: ref.messageId }, - ref.messageId, - ), - ), - Effect.flatMap((raw) => - decodeResult(raw).pipe( - Effect.mapError( - () => - new KiloCloudError({ operation: "getMessageResult", reason: "invalid_response" }), - ), - ), - ), - Effect.flatMap((result) => - result.cloudAgentSessionId !== ref.sessionId || result.messageId !== ref.messageId - ? Effect.fail( - new KiloCloudError({ operation: "getMessageResult", reason: "wrong_owner" }), - ) - : Effect.succeed({ ...result, billingStatus: "unknown" as const }), - ), - ), - interrupt: (ref: KiloCloudRef) => - own(ref).pipe( - Effect.andThen( - Effect.fail(new KiloCloudError({ operation: "interrupt", reason: "unsupported" })), - ), - ), - }; -}; diff --git a/apps/server/src/provider/kilo/KiloCloudError.ts b/apps/server/src/provider/kilo/KiloCloudError.ts new file mode 100644 index 000000000000..94703bf84364 --- /dev/null +++ b/apps/server/src/provider/kilo/KiloCloudError.ts @@ -0,0 +1,18 @@ +import * as Schema from "effect/Schema"; + +/** Failures from Kilo Cloud customer endpoints. None of them proves that billing stopped. */ +export class KiloCloudError extends Schema.TaggedError()("KiloCloudError", { + operation: Schema.String, + reason: Schema.Literals([ + "rejected", + "not_found", + "admission_unknown", + "invalid_response", + "wrong_owner", + "unsupported", + "recovery_incomplete", + "recovery_limit", + ]), + recoveryCause: Schema.optional(Schema.String), + messageId: Schema.optional(Schema.String), +}) {} diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.live.test.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.live.test.ts deleted file mode 100644 index 74b4db8b602a..000000000000 --- a/apps/server/src/provider/kilo/KiloCloudWebClient.live.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; -import { describe } from "vite-plus/test"; -import * as Effect from "effect/Effect"; -import * as Account from "./KiloCloudAccount.ts"; -import * as Cloud from "./KiloCloudWebClient.ts"; - -// Read-only opt-in. This never prepares a sandbox, submits a prompt or answers a request. -const profile = process.env.KILO_CLOUD_TEST_PROFILE; -const sessionId = process.env.KILO_CLOUD_READ_SESSION; -describe.skipIf(!profile || !sessionId)("Kilo Cloud customer read contract", () => { - it.live( - "reads native history and independent sandbox/billing evidence with official CLI login", - () => - Effect.gen(function* () { - const account = yield* Account.make(profile!); - const credentials = yield* account.load; - const client = Cloud.make({ ...credentials, credentials: account.load }); - const session = yield* client.getSession(sessionId!); - const binding: Cloud.CloudBinding = { - accountId: credentials.accountId, - cloudAgentSessionId: session.sessionId, - kiloSessionId: session.kiloSessionId, - worktreeId: session.worktreeId, - repository: session.githubRepo, - branch: session.upstreamBranch ?? "main", - }; - const page = yield* client.history(binding); - assert.equal(page.kiloSessionId, session.kiloSessionId); - assert.isTrue( - page.history?.messages.some( - (message) => - message.info.role === "assistant" && - message.parts.some((part) => part.type === "text" && !!part.text), - ) ?? false, - ); - const sandbox = yield* client.sandbox(binding); - const billing = yield* client.billing(binding); - assert.isTrue(sandbox.observedAt > 0); - assert.isTrue(["session", "payer_shared"].includes(billing.attribution)); - }).pipe(Effect.provide(NodeServices.layer)), - 30_000, - ); -}); diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts index 7ab1e9bbfe84..2ee024a1a8d3 100644 --- a/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.test.ts @@ -8,7 +8,7 @@ import * as Effect from "effect/Effect"; import * as Clock from "effect/Clock"; import * as Redacted from "effect/Redacted"; import * as Cloud from "./KiloCloudWebClient.ts"; -import { KiloCloudError } from "./KiloCloudClient.ts"; +import { KiloCloudError } from "./KiloCloudError.ts"; const cleanups: Array<() => Promise> = []; afterEach(async () => { diff --git a/apps/server/src/provider/kilo/KiloCloudWebClient.ts b/apps/server/src/provider/kilo/KiloCloudWebClient.ts index 65e6589dafe0..1d3ea9388241 100644 --- a/apps/server/src/provider/kilo/KiloCloudWebClient.ts +++ b/apps/server/src/provider/kilo/KiloCloudWebClient.ts @@ -6,7 +6,7 @@ import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { KiloCloudError } from "./KiloCloudClient.ts"; +import { KiloCloudError } from "./KiloCloudError.ts"; // Customer routes used by Kilo's web/mobile clients (Kilo-Org/cloud 78ea0a5e). // workspace_* is the control-plane session; ses_* is its conversation; worktree_* diff --git a/apps/server/src/provider/kilo/KiloProcessCleanup.test.ts b/apps/server/src/provider/kilo/KiloProcessCleanup.test.ts deleted file mode 100644 index fc9c107a0552..000000000000 --- a/apps/server/src/provider/kilo/KiloProcessCleanup.test.ts +++ /dev/null @@ -1,391 +0,0 @@ -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; -import * as Deferred from "effect/Deferred"; -import * as Effect from "effect/Effect"; -import * as Exit from "effect/Exit"; -import * as Fiber from "effect/Fiber"; -import * as FileSystem from "effect/FileSystem"; -import * as PlatformError from "effect/PlatformError"; -import * as TestClock from "effect/testing/TestClock"; -import * as Cleanup from "./KiloProcessCleanup.ts"; - -const stat = (state = "S", start = "123") => { - const fields = Array(20).fill("0"); - fields[0] = state; - fields[2] = "4242"; - fields[19] = start; - return `4242 (fixture child) ${fields.join(" ")}`; -}; -const records = (fs: FileSystem.FileSystem, root: string) => - Effect.gen(function* () { - const directories = yield* fs.readDirectory(`${root}/kilo-cleanup`); - return (yield* Effect.forEach(directories, (name) => - fs.readDirectory(`${root}/kilo-cleanup/${name}`), - )).flat(); - }); -const denied = () => - PlatformError.systemError({ - _tag: "PermissionDenied", - module: "FileSystem", - method: "readFileString", - pathOrDescriptor: "/proc/4242/stat", - }); -const missing = () => - PlatformError.systemError({ - _tag: "NotFound", - module: "FileSystem", - method: "readFileString", - pathOrDescriptor: "/proc/4242/stat", - }); - -it.effect.each(["zombie", "gone", "reused", "permission", "malformed"] as const)( - "verifies %s without treating observation failure as absence", - (outcome) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - let stopped = false; - let repaired = false; - let starts = 0; - const fake: FileSystem.FileSystem = { - ...fs, - readDirectory: (path) => - path === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(path), - readFileString: (path, ...args) => - path !== "/proc/4242/stat" - ? fs.readFileString(path, ...args) - : Effect.suspend(() => { - if (!stopped) return Effect.succeed(stat()); - if (repaired || outcome === "zombie") return Effect.succeed(stat("Z")); - if (outcome === "gone") return Effect.fail(missing()); - if (outcome === "reused") return Effect.succeed(stat("S", "456")); - if (outcome === "permission") return Effect.fail(denied()); - return Effect.succeed("invalid stat"); - }), - }; - yield* Effect.gen(function* () { - const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); - const result = yield* Effect.exit( - cleanup.verify( - 4242, - Effect.sync(() => { - stopped = true; - }), - ), - ); - const bad = outcome === "permission" || outcome === "malformed"; - assert.equal(Exit.isFailure(result), bad); - // A new helper reads the real on-disk record, not the first helper's closure. - const reopened = yield* Cleanup.make({ profile: root, stateDir: root }); - const start = reopened.withStart(Effect.sync(() => ++starts)); - const admission = yield* Effect.exit(start); - assert.equal(Exit.isFailure(admission), bad); - assert.equal(starts, bad ? 0 : 1); - if (bad) { - assert.deepEqual(yield* records(fs, root), ["4242.json"]); - repaired = true; - yield* start; - assert.equal(starts, 1); - } - assert.deepEqual(yield* records(fs, root), []); - }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), -); - -it.effect("timeout retains the recorded identities and recovery requires observed exit", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const stopping = yield* Deferred.make(); - let state = "S"; - const fake: FileSystem.FileSystem = { - ...fs, - readDirectory: (p) => (p === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(p)), - readFileString: (p, ...a) => - p === "/proc/4242/stat" ? Effect.sync(() => stat(state)) : fs.readFileString(p, ...a), - }; - yield* Effect.gen(function* () { - const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); - const fiber = yield* cleanup - .verify(4242, Deferred.succeed(stopping, undefined).pipe(Effect.asVoid)) - .pipe(Effect.forkScoped); - yield* Deferred.await(stopping); - yield* TestClock.adjust("3 seconds"); - assert.isTrue(Exit.isFailure(yield* Fiber.await(fiber))); - assert.deepEqual(yield* records(fs, root), ["4242.json"]); - state = "Z"; - let starts = 0; - yield* cleanup.withStart(Effect.sync(() => starts++)); - assert.equal(starts, 1); - }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), -); - -it.effect("a failed reservation write cannot be bypassed by a successful directory read", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - let writesFail = true; - let state = "S"; - let signals = 0; - let starts = 0; - const fake: FileSystem.FileSystem = { - ...fs, - readDirectory: (p) => (p === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(p)), - readFileString: (p, ...args) => - p === "/proc/4242/stat" ? Effect.sync(() => stat(state)) : fs.readFileString(p, ...args), - writeFileString: (p, ...args) => - writesFail && p.endsWith("4242.json.tmp") - ? Effect.fail(denied()) - : fs.writeFileString(p, ...args), - }; - yield* Effect.gen(function* () { - const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); - yield* cleanup - .verify( - 4242, - Effect.sync(() => signals++), - ) - .pipe(Effect.flip); - writesFail = false; - const replacement = yield* Cleanup.make({ profile: root, stateDir: root }); - yield* replacement.withStart(Effect.sync(() => starts++)).pipe(Effect.flip); - assert.equal(signals, 0); - assert.equal(starts, 0); - // An unrelated profile is not blocked by this profile's storage failure. - const other = yield* Cleanup.make({ - profile: `${root}/other`, - stateDir: root, - }); - yield* other.withStart(Effect.sync(() => starts++)); - assert.equal(starts, 1); - state = "Z"; - yield* replacement.withStart(Effect.sync(() => starts++)); - assert.equal(starts, 2); - }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), -); - -it.effect("incomplete snapshots recover only after a complete quiescent-group observation", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - let unreadable = true; - let state = "S"; - let starts = 0; - const fake: FileSystem.FileSystem = { - ...fs, - readDirectory: (p) => (p === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(p)), - readFileString: (p, ...args) => - p === "/proc/4242/stat" - ? Effect.suspend(() => (unreadable ? Effect.fail(denied()) : Effect.succeed(stat(state)))) - : fs.readFileString(p, ...args), - }; - yield* Effect.gen(function* () { - const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); - yield* cleanup - .verify(4242, Effect.die("must not signal without observation")) - .pipe(Effect.flip); - const reopened = yield* Cleanup.make({ profile: root, stateDir: root }); - const start = reopened.withStart(Effect.sync(() => starts++)); - yield* start.pipe(Effect.flip); - unreadable = false; - yield* start.pipe(Effect.flip); - assert.equal(starts, 0); - state = "Z"; - yield* start; - assert.equal(starts, 1); - assert.deepEqual(yield* records(fs, root), []); - }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), -); - -it.effect( - "first-open profile aliases share pending cleanup and preserve other account admission", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${root}/real`); - yield* fs.symlink(`${root}/real`, `${root}/alias`); - let unreadable = false; - let state = "S"; - const fake: FileSystem.FileSystem = { - ...fs, - readDirectory: (p) => (p === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(p)), - readFileString: (p, ...args) => - p === "/proc/4242/stat" - ? Effect.suspend(() => - unreadable ? Effect.fail(denied()) : Effect.succeed(stat(state)), - ) - : fs.readFileString(p, ...args), - }; - yield* Effect.gen(function* () { - const cleanup = yield* Cleanup.make({ - profile: `${root}/alias/new-profile`, - stateDir: root, - }); - yield* cleanup - .verify( - 4242, - Effect.sync(() => { - unreadable = true; - }), - ) - .pipe(Effect.flip); - const reopened = yield* Cleanup.make({ - profile: `${root}/real/new-profile`, - stateDir: root, - }); - let starts = 0; - yield* reopened.withStart(Effect.sync(() => starts++)).pipe(Effect.flip); - assert.equal(starts, 0); - // Even a malformed pending record in one account must not block another. - const [key] = yield* fs.readDirectory(`${root}/kilo-cleanup`); - const file = `${root}/kilo-cleanup/${key}/4242.json`; - const saved = yield* fs.readFileString(file); - yield* fs.writeFileString(file, "incomplete write"); - const other = yield* Cleanup.make({ profile: `${root}/other`, stateDir: root }); - yield* other.withStart(Effect.sync(() => starts++)); - assert.equal(starts, 1); - yield* fs.writeFileString(file, saved); - unreadable = false; - state = "Z"; - yield* reopened.withStart(Effect.sync(() => starts++)); - assert.equal(starts, 2); - }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), -); - -it.effect("cancellation releases the lock but preserves uncertainty until observed exit", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const entered = yield* Deferred.make(); - let park = true; - let state = "S"; - const fake: FileSystem.FileSystem = { - ...fs, - readDirectory: (p) => - p === "/proc" - ? Effect.suspend(() => - park - ? Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never)) - : Effect.succeed(["4242"]), - ) - : fs.readDirectory(p), - readFileString: (p, ...args) => - p === "/proc/4242/stat" ? Effect.sync(() => stat(state)) : fs.readFileString(p, ...args), - }; - yield* Effect.gen(function* () { - const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); - const fiber = yield* cleanup.verify(4242, Effect.void).pipe(Effect.forkScoped); - yield* Deferred.await(entered); - yield* Fiber.interrupt(fiber); - park = false; - let starts = 0; - const reopened = yield* Cleanup.make({ profile: root, stateDir: root }); - yield* reopened.withStart(Effect.sync(() => starts++)).pipe(Effect.flip); - assert.equal(starts, 0); - state = "Z"; - yield* reopened.withStart(Effect.sync(() => starts++)); - assert.equal(starts, 1); - }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), -); - -it.effect( - "holds the shared gate through observation and exit, without serializing session lifetime", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const scanning = yield* Deferred.make(); - const release = yield* Deferred.make(); - const requested = yield* Deferred.make(); - const observedAlive = yield* Deferred.make(); - let signalled = false; - const events: Array = []; - let state = "S"; - const fake: FileSystem.FileSystem = { - ...fs, - readDirectory: (p) => - p === "/proc" - ? Deferred.succeed(scanning, undefined).pipe( - Effect.andThen(Deferred.await(release)), - Effect.as(["4242"]), - ) - : fs.readDirectory(p), - readFileString: (p, ...args) => - p === "/proc/4242/stat" - ? Effect.sync(() => { - const value = stat(state); - if (state === "Z") events.push("confirmed"); - else if (signalled) Deferred.doneUnsafe(observedAlive, Effect.void); - return value; - }) - : fs.readFileString(p, ...args), - }; - yield* Effect.gen(function* () { - const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); - const other = yield* Cleanup.make({ profile: root, stateDir: root }); - const stopping = yield* cleanup - .verify( - 4242, - Effect.sync(() => { - signalled = true; - }), - ) - .pipe(Effect.forkScoped); - yield* Deferred.await(scanning); - const replacement = yield* Deferred.succeed(requested, undefined).pipe( - Effect.andThen(other.withStart(Effect.sync(() => events.push("spawn")))), - Effect.forkScoped, - ); - yield* Deferred.await(requested); - yield* Deferred.succeed(release, undefined); - yield* Deferred.await(observedAlive); - assert.deepEqual(events, []); - state = "Z"; - yield* TestClock.adjust("10 millis"); - yield* Fiber.join(stopping); - yield* Fiber.join(replacement); - assert.deepEqual(events, ["confirmed", "spawn"]); - yield* other.withStart(Effect.sync(() => events.push("parallel-session"))); - assert.equal(events.at(-1), "parallel-session"); - }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), -); - -it.effect("a partial snapshot write preserves a readable uncertainty record for recovery", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - let writes = 0; - let state = "S"; - const fake: FileSystem.FileSystem = { - ...fs, - readDirectory: (p) => (p === "/proc" ? Effect.succeed(["4242"]) : fs.readDirectory(p)), - readFileString: (p, ...args) => - p === "/proc/4242/stat" ? Effect.sync(() => stat(state)) : fs.readFileString(p, ...args), - writeFileString: (p, ...args) => - Effect.suspend(() => { - if (p.endsWith(".tmp") && ++writes === 2) - return fs.writeFileString(p, "{partial").pipe(Effect.andThen(Effect.fail(denied()))); - return fs.writeFileString(p, ...args); - }), - }; - yield* Effect.gen(function* () { - const cleanup = yield* Cleanup.make({ profile: root, stateDir: root }); - yield* cleanup.verify(4242, Effect.die("no signal after partial write")).pipe(Effect.flip); - const reopened = yield* Cleanup.make({ profile: root, stateDir: root }); - let starts = 0; - const start = reopened.withStart(Effect.sync(() => starts++)); - yield* start.pipe(Effect.flip); - assert.equal(starts, 0); - state = "Z"; - yield* start; - assert.equal(starts, 1); - }).pipe(Effect.provideService(FileSystem.FileSystem, fake)); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), -); diff --git a/apps/server/src/provider/kilo/KiloProcessCleanup.ts b/apps/server/src/provider/kilo/KiloProcessCleanup.ts deleted file mode 100644 index f71849f1bb22..000000000000 --- a/apps/server/src/provider/kilo/KiloProcessCleanup.ts +++ /dev/null @@ -1,185 +0,0 @@ -import * as Crypto from "effect/Crypto"; -import * as Encoding from "effect/Encoding"; -import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Path from "effect/Path"; -import * as Schema from "effect/Schema"; -import * as Semaphore from "effect/Semaphore"; - -export class KiloCleanupError extends Schema.TaggedError()("KiloCleanupError", { - detail: Schema.String, -}) { - override get message() { - return this.detail; - } -} - -const Identity = Schema.Struct({ pid: Schema.Int, startTime: Schema.String }); -const Record = Schema.Struct({ - profile: Schema.String, - // null means observation did not complete; it is not an empty process set. - members: Schema.NullOr(Schema.Array(Identity)), -}); -const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(Record)); -const encode = Schema.encodeEffect(Schema.fromJsonString(Record)); -const gates = new Map }>(); -const failure = () => - new KiloCleanupError({ - detail: - "Kilo process cleanup could not be confirmed. Replacement remains blocked; check process and state-directory access before retrying.", - }); - -/** Linux observation only: not a sandbox, process-tree discovery or atomic signal identity. */ -export const make = Effect.fn("KiloProcessCleanup.make")(function* (input: { - readonly profile: string; - readonly stateDir: string; -}) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - // Create the owned directory before canonicalizing: first-open aliases must use - // the same gate and record namespace as subsequent opens through the real path. - yield* fs.makeDirectory(input.profile, { recursive: true }).pipe(Effect.mapError(failure)); - const profile = yield* fs.realPath(input.profile).pipe(Effect.mapError(failure)); - const crypto = yield* Crypto.Crypto; - const key = yield* crypto - .digest("SHA-256", new TextEncoder().encode(profile)) - .pipe(Effect.map(Encoding.encodeHex), Effect.mapError(failure)); - let gate = gates.get(profile); - if (!gate) { - gate = { lock: Semaphore.makeUnsafe(1), unrecorded: new Set() }; - gates.set(profile, gate); - } - const handoff = gate; - const directory = path.join(input.stateDir, "kilo-cleanup", key); - const observe = (pid: number) => - fs.readFileString(`/proc/${pid}/stat`).pipe( - Effect.flatMap((stat) => { - const fields = stat - .slice(stat.lastIndexOf(")") + 2) - .trim() - .split(/\s+/); - if ( - !stat.startsWith(`${pid} (`) || - !/^[A-Za-z]$/.test(fields[0] ?? "") || - !/^\d+$/.test(fields[2] ?? "") || - !/^\d+$/.test(fields[19] ?? "") - ) - return Effect.fail(failure()); - return Effect.succeed({ - pid, - startTime: fields[19]!, - pgid: Number(fields[2]), - stopped: fields[0] === "Z" || fields[0] === "X" || fields[0] === "x", - }); - }), - Effect.catchTag("PlatformError", (error) => - error.reason._tag === "NotFound" ? Effect.succeed(undefined) : Effect.fail(failure()), - ), - ); - const wait = (members: ReadonlyArray) => - Effect.gen(function* () { - for (;;) { - const statuses = yield* Effect.forEach( - members, - (member) => - observe(member.pid).pipe( - Effect.map( - (current) => - current === undefined || - current.startTime !== member.startTime || - current.stopped, - ), - ), - { concurrency: 8 }, - ); - if (statuses.every(Boolean)) return; - // A bounded condition-driven OS observation, never a fixed cleanup grace assertion. - yield* Effect.sleep("10 millis"); - } - }).pipe(Effect.timeout("2 seconds"), Effect.mapError(failure)); - const snapshot = (pgid: number) => - Effect.gen(function* () { - const names = yield* fs.readDirectory("/proc").pipe(Effect.mapError(failure)); - const observed = yield* Effect.forEach( - names.filter((name) => /^\d+$/.test(name)), - (name) => observe(Number(name)), - { concurrency: 16 }, - ); - return observed.flatMap((item) => (item !== undefined && item.pgid === pgid ? [item] : [])); - }); - // For incomplete snapshots, a successful fresh group scan must prove quiescence. - // This never signals newly discovered PIDs, nor claims to find escaped descendants. - const confirmGroup = (pgid: number) => - snapshot(pgid).pipe( - Effect.flatMap((members) => - members.every((member) => member.stopped) ? Effect.void : Effect.fail(failure()), - ), - ); - const entries = fs.readDirectory(directory).pipe( - Effect.catchTag("PlatformError", (error) => - error.reason._tag === "NotFound" ? Effect.succeed([] as Array) : Effect.fail(error), - ), - Effect.mapError(failure), - ); - const save = (file: string, members: (typeof Record.Type)["members"]) => - Effect.gen(function* () { - const temporary = `${file}.tmp`; - yield* fs - .writeFileString(temporary, yield* encode({ profile, members })) - .pipe(Effect.mapError(failure)); - // Same-directory rename keeps the previous uncertainty record readable if a - // later write is interrupted or partially fails. This is not a power-loss fsync guarantee. - yield* fs.rename(temporary, file).pipe(Effect.mapError(failure)); - }); - const recover = Effect.gen(function* () { - for (const pgid of handoff.unrecorded) { - yield* confirmGroup(pgid); - // A working read is not proof that writes have recovered. - yield* fs.makeDirectory(directory, { recursive: true }).pipe(Effect.mapError(failure)); - const file = path.join(directory, `${pgid}.json`); - yield* save(file, []); - yield* fs.remove(file).pipe(Effect.mapError(failure)); - handoff.unrecorded.delete(pgid); - } - for (const name of yield* entries) { - if (!/^\d+\.json$/.test(name)) continue; - const file = path.join(directory, name); - const record = yield* fs - .readFileString(file) - .pipe(Effect.flatMap(decode), Effect.mapError(failure)); - if (record.profile !== profile) continue; - if (record.members === null) yield* confirmGroup(Number(name.slice(0, -5))); - else yield* wait(record.members); - yield* fs.remove(file).pipe(Effect.mapError(failure)); - } - }); - const verify = (pgid: number, stop: Effect.Effect) => - Effect.gen(function* () { - handoff.unrecorded.add(pgid); - yield* fs.makeDirectory(directory, { recursive: true }).pipe(Effect.mapError(failure)); - const file = path.join(directory, `${pgid}.json`); - // Persist uncertainty before taking a snapshot or delivering any signal. - yield* save(file, null); - handoff.unrecorded.delete(pgid); - const members = (yield* snapshot(pgid)).map(({ pid, startTime }) => ({ pid, startTime })); - yield* save(file, members); - yield* stop; - yield* wait(members); - yield* fs.remove(file).pipe(Effect.mapError(failure)); - }); - return { - // A short start/cleanup gate, not a session-lifetime lock: live sessions may coexist. - withStart: (start: Effect.Effect) => - handoff.lock.withPermit( - recover.pipe(Effect.timeout("5 seconds"), Effect.mapError(failure), Effect.andThen(start)), - ), - verify: (pgid: number, stop: Effect.Effect) => - handoff.lock.withPermit( - verify(pgid, stop).pipe( - Effect.timeout("5 seconds"), - Effect.mapError(failure), - Effect.interruptible, - ), - ), - }; -}); diff --git a/apps/server/src/provider/kilo/KiloRuntime.cleanup.test.ts b/apps/server/src/provider/kilo/KiloRuntime.cleanup.test.ts deleted file mode 100644 index 03dc9b5ce56f..000000000000 --- a/apps/server/src/provider/kilo/KiloRuntime.cleanup.test.ts +++ /dev/null @@ -1,241 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off - real CLI lifecycle and spawn observation. -import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; -import * as NodeChildProcess from "node:child_process"; -import * as NodeURL from "node:url"; -import * as NodeFS from "node:fs"; -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; -import * as Deferred from "effect/Deferred"; -import * as Effect from "effect/Effect"; -import * as Fiber from "effect/Fiber"; -import * as FileSystem from "effect/FileSystem"; -import * as Path from "effect/Path"; -import * as Schema from "effect/Schema"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import * as KiloRuntime from "./KiloRuntime.ts"; - -const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); -const binary = process.env.KILO_BIN; -it.live.skipIf(!binary || HostProcessPlatform.defaultValue() !== "linux")( - "confirms the observed child exit before cleanup returns and a replacement really starts", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-handoff-" }); - const plugin = path.join(root, "fixture.ts"); - const marker = path.join(root, "child"); - const ready = yield* Deferred.make(); - yield* Effect.acquireRelease( - Effect.sync(() => - NodeFS.watch(root, (event, file) => { - if (event === "change" && file === "child") Deferred.doneUnsafe(ready, Effect.void); - }), - ), - (watcher) => Effect.sync(() => watcher.close()), - ); - const childCode = `require('node:fs').writeFileSync(${encodeJson(marker)},String(process.pid));setInterval(()=>{},1000)`; - yield* fs.writeFileString( - plugin, - `import {spawn} from 'node:child_process'; -spawn(${encodeJson(process.execPath)},['-e',${encodeJson(childCode)}],{stdio:'ignore'}); -export const fixture=async()=>({});\n`, - ); - const entered = yield* Deferred.make(); - const release = yield* Deferred.make(); - const replacement = yield* Deferred.make(); - const timeline: Array = []; - let armed = false; - let child = 0; - let identity = ""; - let starts = 0; - const observedFs: FileSystem.FileSystem = { - ...fs, - readDirectory: (directory) => - directory === "/proc" && armed - ? Deferred.succeed(entered, undefined).pipe( - Effect.andThen(Deferred.await(release)), - Effect.andThen(fs.readDirectory(directory)), - ) - : fs.readDirectory(directory), - readFileString: (file, ...args) => - fs.readFileString(file, ...args).pipe( - Effect.tap((stat) => - Effect.sync(() => { - if (!armed || file !== `/proc/${child}/stat`) return; - const fields = stat - .slice(stat.lastIndexOf(")") + 2) - .trim() - .split(/\s+/); - assert.equal(fields[19], identity); - if (fields[0] === "Z" || fields[0] === "X") timeline.push("child-exit-confirmed"); - }), - ), - Effect.tapError((error) => - Effect.sync(() => { - if (armed && file === `/proc/${child}/stat` && error.reason._tag === "NotFound") - timeline.push("child-exit-confirmed"); - }), - ), - ), - }; - const observedSpawner = ChildProcessSpawner.make((command) => - Effect.gen(function* () { - starts++; - if (starts === 2) { - timeline.push("replacement-spawn"); - yield* Deferred.succeed(replacement, undefined); - } - return yield* spawner.spawn(command); - }), - ); - yield* Effect.gen(function* () { - const runtime = yield* KiloRuntime.make({ - instanceId: "handoff", - binaryPath: binary!, - profileDirectory: path.join(root, "profile"), - environment: { - PATH: process.env.PATH, - HTTP_PROXY: process.env.HTTP_PROXY, - HTTPS_PROXY: process.env.HTTPS_PROXY, - NO_PROXY: process.env.NO_PROXY, - NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS, - KILO_DISABLE_PROJECT_CONFIG: "1", - HOME: root, - npm_config_offline: "true", - KILO_DISABLE_MODELS_FETCH: "1", - KILO_DISABLE_DEFAULT_PLUGINS: "1", - KILO_DISABLE_EXTERNAL_SKILLS: "1", - KILO_CONFIG_CONTENT: encodeJson({ plugin: [plugin] }), - }, - }); - const first = yield* runtime.open(root); - yield* first.client.models(); - yield* Deferred.await(ready); - child = Number(yield* fs.readFileString(marker)); - const fields = (yield* fs.readFileString(`/proc/${child}/stat`)) - .split(")") - .at(-1)! - .trim() - .split(/\s+/); - identity = fields[19]!; - assert.notEqual(fields[0], "Z"); - armed = true; - const handoff = yield* first.stop.pipe( - Effect.tap(() => Effect.sync(() => timeline.push("cleanup-return"))), - Effect.andThen(runtime.open(root)), - Effect.forkScoped, - ); - // Old cleanup reaches the real second spawn instead of the observation gate. - const milestone = yield* Effect.raceFirst( - Deferred.await(entered).pipe(Effect.as("observation")), - Deferred.await(replacement).pipe(Effect.as("replacement")), - ); - // Release even when the negative control fails, so no fixture is parked. - yield* Deferred.succeed(release, undefined); - assert.equal(milestone, "observation"); - const second = yield* Fiber.join(handoff); - assert.isTrue(yield* second.isRunning); - assert.equal(starts, 2); - assert.isTrue(timeline.indexOf("child-exit-confirmed") >= 0); - assert.isBelow( - timeline.indexOf("child-exit-confirmed"), - timeline.indexOf("cleanup-return"), - ); - assert.isBelow(timeline.indexOf("cleanup-return"), timeline.indexOf("replacement-spawn")); - armed = false; - yield* second.stop; - }).pipe( - Effect.provideService(FileSystem.FileSystem, observedFs), - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, observedSpawner), - ); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - { timeout: 30000 }, -); - -it.live.skipIf(HostProcessPlatform.defaultValue() !== "linux")( - "recovers a persisted cleanup reservation after its writer crashes in another OS process", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-restart-" }); - const launch = (code: string) => - Effect.acquireRelease( - Effect.gen(function* () { - const ready = yield* Deferred.make(); - const exited = yield* Deferred.make(); - let output = ""; - let stderr = ""; - const child = NodeChildProcess.spawn( - process.execPath, - ["--input-type=module", "-e", code], - { - cwd: NodeURL.fileURLToPath(new URL("../../../", import.meta.url)), - detached: true, - stdio: ["ignore", "pipe", "pipe"], - env: { PATH: process.env.PATH }, - }, - ); - child.stdout.on("data", (chunk) => { - output += String(chunk); - if (output.includes("ready")) Deferred.doneUnsafe(ready, Effect.void); - }); - child.stderr.on("data", (chunk) => { - stderr += String(chunk); - }); - child.once("error", (error) => { - Deferred.doneUnsafe(ready, Effect.die(error)); - Deferred.doneUnsafe(exited, Effect.die(error)); - }); - child.once("exit", (code) => { - Deferred.doneUnsafe(ready, Effect.die(`fixture exited before readiness: ${stderr}`)); - Deferred.doneUnsafe(exited, Effect.succeed(code)); - }); - return { - child, - ready: Deferred.await(ready), - exited: Deferred.await(exited), - output: () => output, - }; - }), - ({ child, exited }) => - Effect.sync(() => { - child.kill("SIGKILL"); - }).pipe(Effect.andThen(exited), Effect.asVoid), - ); - const member = yield* launch("process.stdout.write('ready');setInterval(()=>{},1000)"); - yield* member.ready; - const imports = `import * as Effect from 'effect/Effect'; -import * as NodeServices from '@effect/platform-node/NodeServices'; -import * as Cleanup from ${encodeJson(new URL("./KiloProcessCleanup.ts", import.meta.url).href)}; -`; - const make = `const cleanup = yield* Cleanup.make({profile:${encodeJson(root)},stateDir:${encodeJson(root)}});`; - const writer = yield* launch(`${imports} -await Effect.runPromise(Effect.gen(function*(){${make} -yield* cleanup.verify(${member.child.pid},Effect.sync(()=>process.stdout.write('ready')).pipe(Effect.andThen(Effect.never))); -}).pipe(Effect.provide(NodeServices.layer)));`); - yield* writer.ready; // The real file was saved before verify enters its stop action. - writer.child.kill("SIGKILL"); - yield* writer.exited; - const attempt = `${imports} -await Effect.runPromise(Effect.gen(function*(){${make} -yield* cleanup.withStart(Effect.sync(()=>process.stdout.write('spawn'))); -}).pipe(Effect.provide(NodeServices.layer),Effect.catch(()=>Effect.sync(()=>{process.exitCode=23;}))));`; - const blocked = yield* launch(attempt); - assert.equal(yield* blocked.exited, 23); - assert.equal(blocked.output(), ""); - const [key] = yield* fs.readDirectory(path.join(root, "kilo-cleanup")); - assert.deepEqual(yield* fs.readDirectory(path.join(root, "kilo-cleanup", key!)), [ - `${member.child.pid}.json`, - ]); - member.child.kill("SIGKILL"); - yield* member.exited; // This parent reaps its own child, rather than inferring exit from kill success. - const recovered = yield* launch(attempt); - assert.equal(yield* recovered.exited, 0); - assert.equal(recovered.output(), "spawn"); - assert.deepEqual(yield* fs.readDirectory(path.join(root, "kilo-cleanup", key!)), []); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - { timeout: 15000 }, -); diff --git a/apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs b/apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs deleted file mode 100644 index 2d302448401f..000000000000 --- a/apps/server/src/provider/kilo/KiloRuntime.crash.fixture.mjs +++ /dev/null @@ -1,44 +0,0 @@ -import * as Effect from "effect/Effect"; -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import * as Runtime from "./KiloRuntime.ts"; - -// Report the owned PID immediately for emergency cleanup; readiness is separate. -await Effect.runPromise( - Effect.scoped( - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - let pid; - const runtime = yield* Runtime.make({ - instanceId: "crash-fixture", - binaryPath: process.argv[2], - profileDirectory: process.argv[3], - ...(process.argv[4] ? { processStateDirectory: process.argv[4] } : {}), - environment: { - PATH: process.env.PATH, - HOME: process.argv[3], - KILO_DISABLE_MODELS_FETCH: "1", - KILO_DISABLE_DEFAULT_PLUGINS: "1", - KILO_DISABLE_EXTERNAL_SKILLS: "1", - }, - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, { - ...spawner, - spawn: (...args) => - spawner.spawn(...args).pipe( - Effect.tap((child) => - Effect.sync(() => { - pid = Number(child.pid); - process.send({ type: "spawned", pid }); - }), - ), - ), - }), - ); - const connection = yield* runtime.open(process.argv[3]); - const session = yield* connection.client.create([]); - process.send({ type: "ready", pid, session }); - yield* Effect.never; - }).pipe(Effect.provide(NodeServices.layer)), - ), -); diff --git a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts b/apps/server/src/provider/kilo/KiloRuntime.live.test.ts deleted file mode 100644 index 0731ee7b7b3b..000000000000 --- a/apps/server/src/provider/kilo/KiloRuntime.live.test.ts +++ /dev/null @@ -1,435 +0,0 @@ -// @effect-diagnostics nodeBuiltinImport:off - real owner crash fixture. -import * as NodeChildProcess from "node:child_process"; -import * as NodeEvents from "node:events"; -import * as NodeURL from "node:url"; -import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; -import { ProviderInstanceId, ProviderSessionId, ThreadId } from "@t3tools/contracts"; -import * as Effect from "effect/Effect"; -import * as Exit from "effect/Exit"; -import * as FileSystem from "effect/FileSystem"; -import * as Layer from "effect/Layer"; -import * as Path from "effect/Path"; -import * as Scope from "effect/Scope"; -import * as Schema from "effect/Schema"; -import { describe } from "vite-plus/test"; - -import * as KiloRuntime from "./KiloRuntime.ts"; -import { KiloDriver } from "../Drivers/KiloDriver.ts"; -import * as ServerConfig from "../../config.ts"; -import * as IdAllocator from "../../orchestration-v2/IdAllocator.ts"; - -const binary = process.env.KILO_BIN; -const platform = HostProcessPlatform.defaultValue(); -const decodeGroup = Schema.decodeUnknownEffect( - Schema.fromJsonString(Schema.Struct({ pgid: Schema.Number })), -); -const decodeOwner = Schema.decodeUnknownEffect( - Schema.fromJsonString( - Schema.Struct({ owner: Schema.Struct({ pid: Schema.Number }), pgid: Schema.Number }), - ), -); -const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); -const environment = { - PATH: process.env.PATH, - // Fixtures import only Node builtins; do not let optional plugin package setup - // reach the registry or inherit an external npm configuration from HOME. - npm_config_offline: "true", - HTTP_PROXY: process.env.HTTP_PROXY, - HTTPS_PROXY: process.env.HTTPS_PROXY, - NO_PROXY: process.env.NO_PROXY, - NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS, - KILO_DISABLE_MODELS_FETCH: "1", - KILO_DISABLE_DEFAULT_PLUGINS: "1", - KILO_DISABLE_EXTERNAL_SKILLS: "1", - KILO_DISABLE_PROJECT_CONFIG: "1", -}; - -describe.skipIf(!binary)("KiloRuntime native lifecycle", () => { - it.live( - "loads explicitly trusted repository and external plugins before tool approvals", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-plugin-" }); - const cwd = path.join(root, "checkout"); - const pluginDir = path.join(cwd, ".kilo", "plugins"); - yield* fs.makeDirectory(pluginDir, { recursive: true }); - const marker = path.join(root, "repository-plugin-ran"); - const explicitMarker = path.join(root, "explicit-plugin-ran"); - const body = (target: string) => - `import { writeFileSync } from "node:fs";\nimport { spawn } from "node:child_process";\nconst child = spawn(${encodeJson(process.execPath)}, ["-e", "setInterval(()=>{},1000)"], {stdio:"ignore"});\nwriteFileSync(${encodeJson(target)}, String(child.pid));\nexport const fixture = async () => ({});\n`; - yield* fs.writeFileString(path.join(pluginDir, "unsafe.ts"), body(marker)); - const explicitPlugin = path.join(root, "explicit.ts"); - yield* fs.writeFileString(explicitPlugin, body(explicitMarker)); - // Native configuration is explicitly trusted, independently of tool approvals. - const runtime = yield* KiloRuntime.make({ - instanceId: "plugins", - binaryPath: binary!, - profileDirectory: path.join(root, "profile"), - environment: { - ...environment, - HOME: root, - KILO_DISABLE_PROJECT_CONFIG: "0", - KILO_PURE: "0", - KILO_CONFIG_CONTENT: encodeJson({ plugin: [explicitPlugin] }), - }, - }); - const connection = yield* runtime.open(cwd); - yield* connection.client.models(); - const ref = yield* connection.client.create([ - { permission: "*", pattern: "*", action: "ask" }, - ]); - assert.equal((yield* connection.client.read(ref)).id, ref.sessionId); - assert.isTrue(yield* fs.exists(marker)); - assert.isTrue(yield* fs.exists(explicitMarker)); - if (platform === "linux") { - const ledgerDir = path.join(root, "profile", "t3-processes", "opencode-servers"); - const entry = (yield* fs.readDirectory(ledgerDir))[0]!; - const recorded = yield* decodeGroup( - yield* fs.readFileString(path.join(ledgerDir, entry)), - ); - const descendants = [ - Number(yield* fs.readFileString(marker)), - Number(yield* fs.readFileString(explicitMarker)), - ]; - const observe = (pid: number) => - fs.readFileString(`/proc/${pid}/stat`).pipe( - Effect.map((stat) => { - const fields = stat - .slice(stat.lastIndexOf(")") + 2) - .trim() - .split(/\s+/); - assert.match(fields[19]!, /^\d+$/); - return { startTime: fields[19]!, state: fields[0]! }; - }), - Effect.catchTag("PlatformError", (error) => - error.reason._tag === "NotFound" ? Effect.succeed(undefined) : Effect.fail(error), - ), - ); - const identities = yield* Effect.forEach(descendants, (pid) => - Effect.gen(function* () { - const observed = yield* observe(pid); - assert.isDefined(observed); - assert.notEqual(observed!.state, "Z"); - return { pid, startTime: observed!.startTime }; - }), - ); - // This test proves eventual cleanup of these fixture children, not the - // ordering of replacement. The handoff test verifies that separately. - process.kill(recorded.pgid, "SIGKILL"); - yield* connection.exitCode; - yield* Effect.forEach(identities, (identity) => - Effect.gen(function* () { - for (;;) { - const current = yield* observe(identity.pid); - if ( - current === undefined || - current.startTime !== identity.startTime || - current.state === "Z" || - current.state === "X" - ) - return; - yield* Effect.sleep("10 millis"); - } - }).pipe(Effect.timeout("2 seconds")), - ); - } - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - { timeout: 30000 }, - ); - - it.live( - "retires live clients and rejects saved threads after credentials change in the same profile", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-account-change-" }); - const authDir = path.join(root, "data", "kilo"); - const authFile = path.join(authDir, "auth.json"); - yield* fs.makeDirectory(authDir, { recursive: true }); - const credentials = (key: string) => JSON.stringify({ kilo: { type: "api", key } }); - yield* fs.writeFileString(authFile, credentials("synthetic-account-a")); - const runtime = yield* KiloRuntime.make({ - instanceId: "account-test", - binaryPath: binary!, - profileDirectory: root, - environment: { ...environment, HOME: root }, - }); - const connection = yield* runtime.open(root); - const native = yield* connection.client.create([]); - const create = KiloDriver.create({ - instanceId: ProviderInstanceId.make("account-test"), - displayName: undefined, - enabled: false, - config: { ...KiloDriver.defaultConfig(), binaryPath: binary!, profileDirectory: root }, - environment: Object.entries({ ...environment, HOME: root }).flatMap(([name, value]) => - value === undefined ? [] : [{ name, value, sensitive: false }], - ), - }); - const verify = Effect.gen(function* () { - const first = yield* create; - const request = { - threadId: ThreadId.make("account-test"), - providerSessionId: ProviderSessionId.make("account-test"), - modelSelection: { instanceId: first.instanceId, model: "fixture/test" }, - runtimePolicy: { - runtimeMode: "full-access" as const, - interactionMode: "default" as const, - cwd: root, - }, - }; - const firstSession = yield* first.orchestrationAdapter.openSession(request); - const thread = yield* firstSession.ensureThread(request); - const unchanged = yield* create; - assert.deepStrictEqual(unchanged.continuationIdentity, first.continuationIdentity); - const resumed = yield* unchanged.orchestrationAdapter.openSession(request); - assert.equal( - (yield* resumed.resumeThread({ providerThread: thread })).nativeThreadRef?.nativeId, - thread.nativeThreadRef?.nativeId, - ); - yield* fs.writeFileString(authFile, credentials("synthetic-account-b")); - const failure = yield* connection.client.read(native).pipe(Effect.flip); - assert.equal(failure.reason, "wrong_owner"); - yield* connection.exitCode.pipe(Effect.timeout("5 seconds")); - assert.isFalse(yield* connection.isRunning); - yield* runtime.open(root).pipe(Effect.flip); - const replacement = yield* create; - assert.notEqual( - replacement.continuationIdentity.continuationKey, - first.continuationIdentity.continuationKey, - ); - const secondSession = yield* replacement.orchestrationAdapter.openSession(request); - const rejected = yield* secondSession - .resumeThread({ providerThread: thread }) - .pipe(Effect.flip); - assert.include(rejected.message, "account or configuration changed"); - const fresh = yield* secondSession.ensureThread(request); - assert.notEqual(fresh.nativeThreadRef?.nativeId, thread.nativeThreadRef?.nativeId); - }); - yield* verify.pipe( - Effect.provide( - Layer.mergeAll( - ServerConfig.layerTest(root, { prefix: "t3-kilo-driver-" }), - IdAllocator.layer, - ), - ), - ); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - { timeout: 60000 }, - ); - - it.live("rejects malformed credentials without exposing them or falling back to disk", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-auth-" }); - for (const content of ["", "secret-not-json", "null", "[]"]) { - const failure = yield* KiloRuntime.readAuth(root, { KILO_AUTH_CONTENT: content }).pipe( - Effect.flip, - ); - assert.equal(failure.operation, "authentication"); - assert.notInclude(failure.message, "secret-not-json"); - assert.isUndefined(failure.cause); - } - assert.equal(yield* KiloRuntime.readAuth(root, {}), "{}"); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - - it.live( - "isolates profiles, closes owned processes and resumes after restart", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-runtime-" }); - const cwd = path.join(root, "work"); - yield* fs.makeDirectory(cwd); - const accountScope = yield* Scope.fork(yield* Effect.scope); - const runtime = yield* KiloRuntime.make({ - instanceId: "personal", - binaryPath: binary!, - profileDirectory: path.join(root, "personal"), - environment: { ...environment, HOME: root }, - }).pipe(Effect.provideService(Scope.Scope, accountScope)); - const work = yield* KiloRuntime.make({ - instanceId: "work", - binaryPath: binary!, - profileDirectory: path.join(root, "work-account"), - environment: { ...environment, HOME: root }, - }); - const sessionScope = yield* Scope.fork(yield* Effect.scope); - const first = yield* runtime - .open(cwd) - .pipe(Effect.provideService(Scope.Scope, sessionScope)); - const other = yield* work.open(cwd); - const ref = yield* first.client.create([]); - const foreign = { ...ref, instanceId: "work" }; - yield* other.client.read(foreign).pipe(Effect.flip); - const otherRef = yield* other.client.create([]); - yield* Scope.close(sessionScope, Exit.void); - assert.isFalse(yield* first.isRunning); - assert.isTrue(yield* other.isRunning); - const resumed = yield* runtime.open(cwd); - assert.equal((yield* resumed.client.read(ref)).id, ref.sessionId); - yield* Scope.close(accountScope, Exit.void); - assert.isFalse(yield* resumed.isRunning); - const retired = yield* runtime.open(cwd).pipe(Effect.flip); - assert.equal(retired.operation, "open"); - assert.isTrue(yield* other.isRunning); - assert.equal((yield* other.client.read(otherRef)).id, otherRef.sessionId); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - { timeout: 30000 }, - ); - - for (const globalLedger of [false, true]) - it.live.skipIf(platform !== "linux")( - `reaps a real Kilo owner crash, including moved profiles: globalLedger=${globalLedger}`, - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-crash-" }); - const profile = path.join(root, "profile"); - yield* fs.makeDirectory(profile); - const processStateDirectory = path.join(root, "server-state"); - const ledgerDir = path.join( - globalLedger ? processStateDirectory : path.join(profile, "t3-processes"), - "opencode-servers", - ); - let group: number | undefined; - const owner = yield* Effect.acquireRelease( - Effect.sync(() => - NodeChildProcess.spawn( - process.execPath, - [ - NodeURL.fileURLToPath( - new URL("./KiloRuntime.crash.fixture.mjs", import.meta.url), - ), - binary!, - profile, - ...(globalLedger ? [processStateDirectory] : []), - ], - { stdio: ["ignore", "ignore", "ignore", "ipc"] }, - ), - ), - (child) => - Effect.sync(() => { - child.kill("SIGKILL"); - if (group !== undefined) { - try { - process.kill(-group, "SIGKILL"); - } catch { - /* already stopped */ - } - } - }), - ); - const message = yield* Effect.promise( - () => - new Promise<{ - pid: number; - session: { instanceId: string; sessionId: string; directory: string }; - }>((resolve, reject) => { - owner.on("message", (value) => { - const message = value as { - type: string; - pid: number; - session: { instanceId: string; sessionId: string; directory: string }; - }; - group = message.pid; - if (message.type === "ready") resolve(message); - }); - owner.once("exit", () => - reject(new Error("Kilo crash fixture exited before readiness")), - ); - owner.once("error", reject); - }), - ); - const entries = yield* fs.readDirectory(ledgerDir); - assert.equal(entries.length, 1); - const recorded = yield* decodeOwner( - yield* fs.readFileString(path.join(ledgerDir, entries[0]!)), - ); - assert.equal(recorded.owner.pid, owner.pid); - assert.equal(recorded.pgid, group); - const exited = NodeEvents.EventEmitter.once(owner, "exit"); - owner.kill("SIGKILL"); - yield* Effect.promise(() => exited); - const running = (pid: number) => - fs.readFileString(`/proc/${pid}/stat`).pipe( - Effect.map((stat) => !stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z")), - Effect.orElseSucceed(() => false), - ); - assert.isTrue(yield* running(message.pid)); - const replacementProfile = globalLedger ? path.join(root, "moved-profile") : profile; - if (globalLedger) { - yield* fs.rename(profile, replacementProfile); - yield* fs.makeDirectory(profile); - } - const restarted = yield* KiloRuntime.make({ - instanceId: "crash-fixture", - binaryPath: binary!, - profileDirectory: replacementProfile, - ...(globalLedger ? { processStateDirectory } : {}), - environment: { ...environment, HOME: profile }, - }); - assert.isFalse(yield* running(message.pid)); - assert.deepEqual(yield* fs.readDirectory(ledgerDir), []); - const fresh = yield* restarted.open(profile); - if (globalLedger) { - const newSession = yield* fresh.client.create([]); - assert.notEqual(newSession.sessionId, message.session.sessionId); - } else - assert.equal((yield* fresh.client.read(message.session)).id, message.session.sessionId); - assert.isTrue(yield* fresh.isRunning); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - { timeout: 30000 }, - ); - - it.live( - "cleans failed startup and can open a fresh process afterward", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-kilo-startup-" }); - const bad = yield* KiloRuntime.make({ - instanceId: "broken", - binaryPath: path.join(root, "missing"), - profileDirectory: root, - environment: { ...environment, HOME: root }, - }); - const failure = yield* bad.open(root).pipe(Effect.flip); - assert.equal(failure.operation, "spawn"); - const earlyExit = path.join(root, "early-exit"); - yield* fs.writeFileString( - earlyExit, - "#!/bin/sh\necho do-not-leak-this-diagnostic >&2\nexit 7\n", - ); - yield* fs.chmod(earlyExit, 0o700); - const exiting = yield* KiloRuntime.make({ - instanceId: "early", - binaryPath: earlyExit, - profileDirectory: root, - environment: { ...environment, HOME: root }, - }); - const earlyFailure = yield* exiting.open(root).pipe(Effect.flip); - assert.equal(earlyFailure.operation, "startup"); - assert.notInclude(earlyFailure.message, "do-not-leak"); - assert.include(earlyFailure.message, "code 7"); - const good = yield* KiloRuntime.make({ - instanceId: "working", - binaryPath: binary!, - profileDirectory: root, - environment: { ...environment, HOME: root }, - }); - const connection = yield* good.open(root); - assert.isTrue(yield* connection.isRunning); - yield* connection.client.create([]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - { timeout: 30000 }, - ); -}); diff --git a/apps/server/src/provider/kilo/KiloRuntime.ts b/apps/server/src/provider/kilo/KiloRuntime.ts index 5ec078484776..e234ace57f3b 100644 --- a/apps/server/src/provider/kilo/KiloRuntime.ts +++ b/apps/server/src/provider/kilo/KiloRuntime.ts @@ -14,7 +14,6 @@ import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { resolveSpawnCommand } from "@t3tools/shared/shell"; import { signalProcessGroup } from "../../process/processGroup.ts"; -import * as KiloProcessCleanup from "./KiloProcessCleanup.ts"; import * as KiloSessionClient from "./KiloSessionClient.ts"; import * as ServerLedger from "../OpenCodeServerLedger.ts"; @@ -93,8 +92,8 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { readonly profileDirectory: string; readonly environment: NodeJS.ProcessEnv; readonly authContent?: string; - /** Stable T3 state directory; process ownership must survive profile removal. */ - readonly processStateDirectory?: string; + /** T3's state directory, where OpenCodeServerLedger records owned processes for the boot reaper. */ + readonly processStateDirectory: string; }) { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const fs = yield* FileSystem.FileSystem; @@ -105,24 +104,8 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { const fail = (operation: string, detail: string) => (cause: unknown) => new KiloRuntimeError({ operation, detail, cause }); const profile = path.resolve(input.profileDirectory); - const processCleanup = - platform === "linux" - ? yield* KiloProcessCleanup.make({ - profile, - stateDir: input.processStateDirectory ?? path.join(profile, "t3-processes"), - }).pipe(Effect.mapError(fail("cleanup", "Could not prepare Kilo process cleanup."))) - : undefined; - const ledger = yield* ServerLedger.make({ - stateDir: input.processStateDirectory ?? path.join(profile, "t3-processes"), - }); - // Await profile handoff before any caller can spawn a replacement. Removed - // profiles are also covered by OpenCodeServerLedger.layer's boot reaper, - // which shares this stateDir/opencode-servers directory. - yield* ledger.reapOrphans; - if (input.processStateDirectory) { - const legacy = yield* ServerLedger.make({ stateDir: path.join(profile, "t3-processes") }); - yield* legacy.reapOrphans; - } + // OpenCodeServerLedger.layer reaps this directory when T3 boots after a crash. + const ledger = yield* ServerLedger.make({ stateDir: input.processStateDirectory }); const authContent = input.authContent ?? (yield* readAuth(profile, input.environment)); const environment: NodeJS.ProcessEnv = { ...input.environment, @@ -195,7 +178,7 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { ); // Forget only after the owned group is stopped, including failed readiness. const ledgerScope = yield* Scope.fork(scope); - const spawn = spawner + const child = yield* spawner .spawn( ChildProcess.make(command.command, command.args, { cwd: directory, @@ -206,16 +189,6 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { }), ) .pipe(Effect.mapError(fail("spawn", "Could not start Kilo. Check the binary path."))); - const child = yield* (processCleanup ? processCleanup.withStart(spawn) : spawn).pipe( - Effect.mapError((cause) => - isRuntimeError(cause) - ? cause - : fail( - "cleanup", - "Kilo could not start while previous process cleanup is unconfirmed.", - )(cause), - ), - ); // Only this captured process group is signalled. No process-name matching. const signal = Effect.uninterruptible( platform === "win32" @@ -241,39 +214,15 @@ export const make = Effect.fn("KiloRuntime.make")(function* (input: { ), ), ); - let verified = false; - const cleanup = yield* Effect.cached( - (processCleanup - ? processCleanup.verify( - Number(child.pid), - signal.pipe( - // Await/reap our own child as well as observing non-child members. - // This wait stays inside the bounded, interruptible verification. - // The Node spawner reports signal termination as an exitCode - // error after the actual exit event. PID observation still follows. - Effect.andThen(child.exitCode.pipe(Effect.ignore)), - ), - ) - : signal - ).pipe( - Effect.tap(() => - Effect.sync(() => { - verified = true; - }), - ), - Effect.orDie, - ), - ); + const cleanup = yield* Effect.cached(signal); + // Registered before recording, so an interrupt while the ledger writes still stops the group. yield* Effect.addFinalizer(() => cleanup); const forget = yield* ledger.track({ pid: Number(child.pid), port: 0, args: ["serve", "--hostname=127.0.0.1", "--port=0"], }); - yield* Scope.addFinalizer( - ledgerScope, - Effect.suspend(() => (verified ? forget : Effect.void)), - ); + yield* Scope.addFinalizer(ledgerScope, forget); const guard = checkAuth.pipe(Effect.onError(() => cleanup)); // Observe idle or in-flight account replacement as well as request boundaries. // Never read credential files once per SSE event. diff --git a/apps/server/src/provider/kilo/KiloSessionClient.live.test.ts b/apps/server/src/provider/kilo/KiloSessionClient.live.test.ts deleted file mode 100644 index 38f2b5387cf7..000000000000 --- a/apps/server/src/provider/kilo/KiloSessionClient.live.test.ts +++ /dev/null @@ -1,271 +0,0 @@ -// @effect-diagnostics globalTimers:off - bounds a native process startup; assertions wait on events, never sleeps. -// @effect-diagnostics nodeBuiltinImport:off - exercises the real SDK over Node HTTP and native CLI processes. -/** KILO_BIN=/path/to/kilo vp test run . No model or cloud task is run. */ -import * as NodeChildProcess from "node:child_process"; -import * as NodeFSP from "node:fs/promises"; -import * as NodeOS from "node:os"; -import * as NodePath from "node:path"; -import * as NodeEvents from "node:events"; -import * as NodeHttp from "node:http"; -import { createKiloClient } from "@kilocode/sdk/v2"; -import * as Effect from "effect/Effect"; -import * as Stream from "effect/Stream"; -import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; -import { afterAll, beforeAll, describe, expect, it } from "vite-plus/test"; - -import * as KiloSessionClient from "./KiloSessionClient.ts"; - -const binary = process.env.KILO_BIN; -const run = Effect.runPromise; - -describe.runIf(binary !== undefined)("Kilo 7.8.3 native local sessions", () => { - let root: string; - let url: string; - let platform: string; - let child: NodeChildProcess.ChildProcess | undefined; - let exited: Promise | undefined; - let inferenceRequests = 0; - const inference = NodeHttp.createServer((_req, res) => { - inferenceRequests++; - res.writeHead(500); - res.end(); - }); - - beforeAll(async () => { - platform = await run(HostProcessPlatform); - root = await NodeFSP.mkdtemp( - NodePath.join(process.env.KILO_TEST_ROOT ?? NodeOS.tmpdir(), "t3-kilo-"), - ); - await Promise.all( - ["a", "b", "config", "data", "cache", "state"].map((p) => - NodeFSP.mkdir(NodePath.join(root, p)), - ), - ); - inference.listen(0, "127.0.0.1"); - await NodeEvents.EventEmitter.once(inference, "listening"); - const address = inference.address(); - if (address === null || typeof address === "string") - throw new Error("No inference fixture listener"); - child = NodeChildProcess.spawn(binary!, ["serve", "--hostname=127.0.0.1", "--port=0"], { - cwd: root, - detached: platform !== "win32", - env: { - PATH: process.env.PATH, - HTTP_PROXY: process.env.HTTP_PROXY, - HTTPS_PROXY: process.env.HTTPS_PROXY, - NO_PROXY: process.env.NO_PROXY, - NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS, - SSL_CERT_FILE: process.env.SSL_CERT_FILE, - XDG_CONFIG_HOME: NodePath.join(root, "config"), - XDG_DATA_HOME: NodePath.join(root, "data"), - XDG_CACHE_HOME: NodePath.join(root, "cache"), - XDG_STATE_HOME: NodePath.join(root, "state"), - KILO_SERVER_PASSWORD: "local-test-only", - KILO_DISABLE_AUTOUPDATE: "1", - KILO_DISABLE_MODELS_FETCH: "1", - KILO_DISABLE_DEFAULT_PLUGINS: "1", - KILO_DISABLE_EXTERNAL_SKILLS: "1", - KILO_DISABLE_PROJECT_CONFIG: "1", - KILO_CONFIG_CONTENT: JSON.stringify({ - plugin: [], - provider: { - fixture: { - npm: "@ai-sdk/openai-compatible", - name: "Offline fixture", - options: { baseURL: `http://127.0.0.1:${address.port}/v1` }, - models: { test: { name: "Offline", limit: { context: 10000, output: 1000 } } }, - }, - }, - }), - }, - stdio: ["ignore", "pipe", "pipe"], - }); - exited = NodeEvents.EventEmitter.once(child, "exit"); - url = await new Promise((resolve, reject) => { - let output = ""; - const timeout = setTimeout(() => reject(new Error("Kilo startup timed out")), 25000); - child!.on("error", (error) => { - clearTimeout(timeout); - reject(error); - }); - child!.on("exit", (code) => { - clearTimeout(timeout); - reject(new Error(`Kilo exited: ${code}`)); - }); - child!.stderr!.on("data", () => {}); - child!.stdout!.on("data", (chunk) => { - output = (output + String(chunk)).slice(-65536); - const match = /kilo server listening on (http:\/\/\S+)/.exec(output); - if (match) { - clearTimeout(timeout); - resolve(match[1]!); - } - }); - }); - }, 30000); - - afterAll(async () => { - if (child?.pid !== undefined && child.exitCode === null) { - if (platform === "win32") child.kill("SIGKILL"); - else process.kill(-child.pid, "SIGKILL"); - await exited; - } - inference.closeAllConnections(); - await new Promise((resolve) => inference.close(() => resolve())); - if (root) await NodeFSP.rm(root, { recursive: true, force: true }); - }); - - const connect = (name: "a" | "b") => - run( - KiloSessionClient.make({ - instanceId: `instance-${name}`, - directory: NodePath.join(root, name), - baseUrl: url, - serverPassword: "local-test-only", - }), - ); - - it("creates concurrent native sessions, reads saved history, resumes and forks without inference", async () => { - const [a, b] = await Promise.all([connect("a"), connect("b")]); - const [ra, rb] = await Promise.all([run(a.create([])), run(b.create([]))]); - expect(ra.sessionId).not.toBe(rb.sessionId); - expect((await run(a.read(ra))).directory).toBe(NodePath.join(root, "a")); - expect((await run(b.read(rb))).directory).toBe(NodePath.join(root, "b")); - const native = createKiloClient({ - baseUrl: url, - directory: NodePath.join(root, "a"), - throwOnError: true, - headers: { Authorization: `Basic ${Buffer.from("kilo:local-test-only").toString("base64")}` }, - }); - // noReply is implemented before the model loop in Kilo 7.8.3 SessionPrompt.prompt. - // Synchronous admission creates real stored messages without any inference calls. - const first = await native.session.prompt({ - sessionID: ra.sessionId, - noReply: true, - model: { providerID: "fixture", modelID: "test" }, - parts: [{ type: "text", text: "first" }], - }); - const second = await native.session.prompt({ - sessionID: ra.sessionId, - noReply: true, - model: { providerID: "fixture", modelID: "test" }, - parts: [{ type: "text", text: "second" }], - }); - const resumed = await connect("a"); - expect((await run(resumed.history(ra))).map((m) => m.info.id)).toEqual([ - first.data!.info.id, - second.data!.info.id, - ]); - expect(await run(b.history(rb))).toEqual([]); - const fork = await run(resumed.fork(ra, second.data!.info.id)); - expect( - (await run(resumed.history(fork))).map((m) => - m.parts.filter((p) => p.type === "text").map((p) => p.text), - ), - ).toEqual([["first"]]); - await run(resumed.revert(ra, second.data!.info.id)); - expect((await run(resumed.read(ra))).revert?.messageID).toBe(second.data!.info.id); - await run(resumed.abort(ra)); - expect((await run(b.read(rb))).id).toBe(rb.sessionId); - expect(inferenceRequests).toBe(0); - }, 30000); - - it("streams actual native message events past sync envelopes without inference", async () => { - const subscribed = Promise.withResolvers(); - // Observe the upstream response to establish stream readiness without timer polling. - const proxy = NodeHttp.createServer((req, res) => { - const upstream = NodeHttp.request( - new URL(req.url!, url), - { - method: req.method, - headers: req.headers, - }, - (response) => { - res.writeHead(response.statusCode!, response.headers); - response.pipe(res); - if (req.url!.startsWith("/event")) subscribed.resolve(); - }, - ); - upstream.on("error", () => res.destroy()); - res.on("close", () => upstream.destroy()); - req.pipe(upstream); - }); - proxy.listen(0, "127.0.0.1"); - await NodeEvents.EventEmitter.once(proxy, "listening"); - const address = proxy.address(); - if (address === null || typeof address === "string") throw new Error("No proxy listener"); - const controller = new AbortController(); - try { - const directory = NodePath.join(root, "a"); - const client = await run( - KiloSessionClient.make({ - instanceId: "native-stream", - directory, - baseUrl: `http://127.0.0.1:${address.port}`, - serverPassword: "local-test-only", - }), - ); - const ref = await run(client.create([])); - const received = run( - client.events(ref).pipe( - Stream.filter((event) => event.type === "message.part.updated"), - Stream.take(1), - Stream.runCollect, - Effect.scoped, - ), - { signal: controller.signal }, - ); - // Always install a rejection handler before the independent native request. - const settled = received.then( - (events) => ({ events }), - (error: unknown) => ({ error }), - ); - await subscribed.promise; - const native = createKiloClient({ - baseUrl: url, - directory, - throwOnError: true, - headers: { - Authorization: `Basic ${Buffer.from("kilo:local-test-only").toString("base64")}`, - }, - }); - const message = await native.session.prompt({ - sessionID: ref.sessionId, - noReply: true, - model: { providerID: "fixture", modelID: "test" }, - parts: [{ type: "text", text: "native stream" }], - }); - const result = await settled; - if ("error" in result) throw result.error; - expect(result.events).toHaveLength(1); - const event = result.events[0]!; - expect(event.type).toBe("message.part.updated"); - if (event.type === "message.part.updated") { - expect(event.properties.part.sessionID).toBe(ref.sessionId); - expect(event.properties.part.messageID).toBe(message.data!.info.id); - } - expect(inferenceRequests).toBe(0); - } finally { - controller.abort(); - proxy.closeAllConnections(); - await new Promise((resolve) => proxy.close(() => resolve())); - } - }, 15000); - - it("rejects wrong auth and foreign native ids using the actual server", async () => { - const error = await run( - KiloSessionClient.make({ - instanceId: "bad", - directory: root, - baseUrl: url, - serverPassword: "wrong", - }).pipe(Effect.flip), - ); - expect(error.reason).toBe("request_failed"); - const [a, b] = await Promise.all([connect("a"), connect("b")]); - const foreign = await run(b.create([])); - const forged = { ...foreign, instanceId: "instance-a", directory: NodePath.join(root, "a") }; - expect((await run(a.abort(forged).pipe(Effect.flip))).reason).toBe("wrong_owner"); - expect((await run(b.read(foreign))).id).toBe(foreign.sessionId); - }, 15000); -}); diff --git a/apps/server/src/provider/kilo/KiloSessionClient.ts b/apps/server/src/provider/kilo/KiloSessionClient.ts index 404bb78b99f1..05cbd3b85ddc 100644 --- a/apps/server/src/provider/kilo/KiloSessionClient.ts +++ b/apps/server/src/provider/kilo/KiloSessionClient.ts @@ -107,7 +107,6 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { readonly directory: string; readonly baseUrl: string; readonly serverPassword?: string; - readonly serverUsername?: string; readonly beforeRequest?: Effect.Effect; }) { const client = createKiloClient({ @@ -120,7 +119,7 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { ? {} : { headers: { - Authorization: `Basic ${Buffer.from(`${input.serverUsername ?? "kilo"}:${input.serverPassword}`).toString("base64")}`, + Authorization: `Basic ${Buffer.from(`kilo:${input.serverPassword}`).toString("base64")}`, }, }), }); @@ -180,11 +179,14 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { run: (signal: AbortSignal) => Promise<{ data?: A }>, ) => read(ref).pipe(Effect.andThen(request(operation, run))); - const reference = (session: { id: string; directory: string }): KiloSessionRef => ({ - instanceId: input.instanceId, - directory: session.directory, - sessionId: session.id, - }); + const ownedReference = (operation: string) => (session: unknown) => + isSessionOwner(session) && session.directory === input.directory + ? Effect.succeed({ + instanceId: input.instanceId, + directory: session.directory, + sessionId: session.id, + }) + : Effect.fail(new KiloSessionError({ operation, reason: "wrong_owner" })); const ownerCheck = (ref: KiloSessionRef, signal: AbortSignal) => { const owners = new Map([[ref.sessionId, true]]); @@ -239,15 +241,7 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { ) => request("session.create", (signal) => client.session.create(permission === undefined ? {} : { permission }, { signal }), - ).pipe( - Effect.flatMap((session) => - isSessionOwner(session) && session.directory === input.directory - ? Effect.succeed(reference(session)) - : Effect.fail( - new KiloSessionError({ operation: "session.create", reason: "wrong_owner" }), - ), - ), - ), + ).pipe(Effect.flatMap(ownedReference("session.create"))), read, history: (ref: KiloSessionRef) => owned(ref, "session.messages", (signal) => @@ -259,21 +253,7 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { { sessionID: ref.sessionId, ...(messageID === undefined ? {} : { messageID }) }, { signal }, ), - ).pipe( - Effect.flatMap((session) => - isSessionOwner(session) && session.directory === input.directory - ? Effect.succeed(reference(session)) - : Effect.fail( - new KiloSessionError({ operation: "session.fork", reason: "wrong_owner" }), - ), - ), - ), - // Native revert can modify files. The orchestration adapter must coordinate it with - // T3 checkpoints and exclude concurrent writers before exposing this operation. - revert: (ref: KiloSessionRef, messageID: string) => - owned(ref, "session.revert", (signal) => - client.session.revert({ sessionID: ref.sessionId, messageID }, { signal }), - ), + ).pipe(Effect.flatMap(ownedReference("session.fork"))), prompt: ( ref: KiloSessionRef, prompt: Omit< @@ -362,15 +342,7 @@ export const make = Effect.fn("KiloSessionClient.make")(function* (input: { abort: (ref: KiloSessionRef) => owned(ref, "session.abort", (signal) => client.session.abort({ sessionID: ref.sessionId }, { signal }), - ).pipe( - Effect.flatMap((accepted) => - accepted === true - ? Effect.void - : Effect.fail( - new KiloSessionError({ operation: "session.abort", reason: "invalid_response" }), - ), - ), - ), + ).pipe(Effect.flatMap(acknowledge("session.abort"))), replyPermission: ( ref: KiloSessionRef, requestID: string, diff --git a/docs/README.md b/docs/README.md index dd59f4417bbb..7c30034a43fd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,7 +19,7 @@ - [Remote access](./user/remote-access.md) - [Running in the background](./user/background-service.md) - [Updating T3 Code](./user/updating.md) -- Provider guides: [Codex](./user/providers-codex.md) · [Claude](./user/providers-claude.md) · [OpenCode](./user/providers-opencode.md) · [Antigravity](./user/providers-antigravity.md) · [Pi](./user/providers-pi.md) +- Provider guides: [Codex](./user/providers-codex.md) · [Claude](./user/providers-claude.md) · [OpenCode](./user/providers-opencode.md) · [Antigravity](./user/providers-antigravity.md) · [Pi](./user/providers-pi.md) · [Kilo](./user/providers-kilo.md) --- diff --git a/docs/user/install.md b/docs/user/install.md index 980837199e9b..e3347decab1f 100644 --- a/docs/user/install.md +++ b/docs/user/install.md @@ -162,7 +162,7 @@ their original values. For provider-specific setup and accounts, see [Codex](./providers-codex.md), [Claude](./providers-claude.md), [OpenCode](./providers-opencode.md), -[Antigravity](./providers-antigravity.md), and [Pi](./providers-pi.md). +[Antigravity](./providers-antigravity.md), [Pi](./providers-pi.md), and [Kilo](./providers-kilo.md). ## Next steps diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md index 78815c157b21..c4e516c07caa 100644 --- a/docs/user/providers-kilo.md +++ b/docs/user/providers-kilo.md @@ -1,133 +1,63 @@ # Kilo -Add **Kilo** in Settings > Providers, select the Kilo CLI 7.8.3 binary and an -account profile, then refresh the provider. Profiles isolate credentials and native -session storage. Sign in with the official Kilo CLI separately; T3 never signs in -automatically. Changing credentials retires the old runtime and requires a new -thread. Saved history remains available. - -## Local configuration and approvals - -Like T3's OpenCode provider, Kilo trusts native runtime configuration, plugins and -MCP servers. Only open repositories and profiles whose configuration you trust. -Trusted native configuration and plugins can access this profile's credentials. -Configured MCP processes or connections can start before a model tool call or -approval. Supervised and Plan modes govern supported tool calls; they are not an -OS sandbox and do not isolate native configuration or MCP initialization. - -Kilo 7.8.3 loads legacy `.kilo/mcp.json` and `.kilocode/mcp.json` even when -`KILO_DISABLE_PROJECT_CONFIG` is enabled. `KILO_PURE` suppresses external plugins, -not all MCP loading. T3 does not force either flag as a security boundary, rewrite -configuration, or patch the installed runtime. Explicit native settings remain -trusted. Background subagents stay disabled. Foreground child agents require Full -access with the default interaction mode. Plan mode disables them, even with Full -access, because Kilo does not inherit parent `ask` rules reliably. - -T3 stops owned processes on normal shutdown. On Linux and macOS, it records their -process identity in T3's state directory and reaps processes from a dead T3 owner -on server startup, including when that account profile was removed or moved. Cleanup checks the recorded PID, start time, command and owner; -it never kills by executable name. Crash recovery on macOS and native Windows -process cleanup have not been verified in this environment. Older profile-local -records are recovered only if that original profile is reopened. Deleting or moving -T3's own state directory can lose cleanup records; T3 never guesses ownership from -a process name. - -On Linux, normal cleanup snapshots the current process group before signalling. -T3 waits for each recorded PID/start-time identity to disappear or reach a -non-executing zombie/dead state before cleanup returns. Its owned child exit is -reaped through the process spawner. Within one T3 server, a per-profile gate covers cleanup and this -verification, and is checked before another local process starts; it does not -prevent concurrent live sessions or lock out another T3 server. Observation errors and timeouts leave a pending -record and block new starts for that profile. Retry can clear it only after a -successful exit observation, not merely because time elapsed. Incomplete snapshots -require a complete fresh scan showing no executing members in the original group. - -This is not a supervisor or sandbox. Enumeration is not atomic: newly forked or -escaped descendants, process-group changes and PID reuse between a native signal -check and delivery are not fully contained. The stronger exit observation is -Linux-only; macOS and Windows retain their existing cleanup behavior. Persisted -pending records survive an ordinary T3 restart if the state directory remains -intact. Failed initial writes only retain uncertainty in memory; power loss, -corrupted/deleted records and crashes before cleanup starts do not gain a stronger -guarantee than the existing orphan reaper. T3 does not rewrite native MCP or plugin -configuration to enforce this lifecycle boundary. - -| Capability | Local Kilo | Kilo Cloud | -| ------------------------------------------ | -------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | -| Prompts and follow-up | Native streaming, tools and reasoning | Full access; history updates, no token-streaming claim | -| Concurrent sessions | Separate processes and account profiles | Separate task identities and remote worktrees; runs alongside local sessions | -| History and recovery | Native history and resume | Durable admission and result recovery; no blind paid resubmission | -| Stop | Native abort and owned-process cleanup | Inference interrupt while running; local retrieval cancellation after remote completion | -| Approvals and questions | Native supported tool approvals and questions | Handles emitted interactions; cannot enforce restricted policy | -| Rewind, fork, checkpoints, text generation | Integrated with native sessions and T3 checkpoints | Not supported | -| Subagents | Foreground Full access and default mode only; restricted/Plan denied | Remote Full access may run them; child history not integrated | -| Clients | Web/desktop/mobile selection, models, status and controls | Account/repository/model settings and task status; local workspace actions disabled | - -## Cloud execution and costs - -Add a separate **Kilo Cloud** instance in Settings > Providers. Select a profile -signed in through the official Kilo login, an accessible GitHub repository, its -branch and a model. Personal accounts are supported. Enabling paid cloud execution -allows prompts and that remote repository to be sent to Kilo. T3 never uploads -local checkout files or uncommitted changes. Each cloud thread has a remote worktree. -Start cloud threads at the project root. T3 skips local setup scripts and managed -folders, and rejects local worktree strategies for cloud launches and setup retries. - -Cloud requires **Full access**. The deployed runtime does not apply custom agent -permissions, so T3 rejects restricted and Plan modes before paid admission. Remote -shell commands, edits and subagents can run. Automatic commits are disabled, but -this is not a read-only execution policy. Profiles with inherited setup commands, -MCP, skills, agents or environment variables are rejected before a new cloud task. -The local trust choice does not relax this cloud restriction. - -A cloud thread cannot use local attachments, terminals, Git actions, checkpoints, -rewind, forks or background text generation. Switching accounts does not transfer -existing tasks or stop them. Reconnecting uses the saved task identity. Uncertain -admission is reconciled through paginated customer APIs, never automatically resent. - -Cloud tasks spend Kilo credit for inference and sandbox use. Inference interruption, -a closed stream, remote completion and sandbox sleep are separate events. Task, -result, sandbox and compute status are shown separately. Compute estimates can cover -a shared account sandbox and are not a per-task invoice. Unknown or settling status -does not mean billing has stopped. T3 does not top up credit or force sandbox sleep. - -## Results after remote completion - -The customer `workspace_` API reports execution status separately from history. -T3 marks a completed task `awaiting_result` until it retrieves output correlated to -the original message, account, worktree and native session. A completed, textless -assistant or terminal tool-only outcome is valid; unrelated replies, unfinished -tools and outstanding interactions cannot finish the local turn. - -Retrieval reads at most four pages per attempt, with a 100-cursor cycle limit, -backoff up to 30 seconds and a five-minute recovery window. Progress, next attempt -and deadline survive restart. A confirmed outstanding interaction gives the user -time to respond and renews that window. Missing output after the window produces a -specific local result-retrieval failure while preserving remote `completed`. -Reopening history can retrieve a late result without restarting the failed turn, -duplicating its messages or submitting a new paid task. - -Stop while `awaiting_result` cancels local result retrieval. It does not send a -remote interrupt or claim the sandbox is sleeping. Stop during running inference -requests remote interruption and waits for confirmation; billing remains separate. - -Local/cloud concurrency and recovery are covered by actual local CLI sessions and -loopback customer-contract tests. These do not replace live verification of every -deployed cloud behavior or native platform testing. - -If preflight fails before the paid request is attempted, T3 ends that turn locally -and permits an explicit new turn. Interrupted or older admission records without -proof of that boundary remain uncertain. A timeout or an incomplete search never -permits automatic resubmission. Repeatedly unreadable admission candidates pause -automatic scanning; reopening history retries only reads. A search also stops at -100 history pages. Reopening cannot bypass that limit; resolving such an operation -requires provider support, not another submission. Journal failures pause recovery -in memory if the pause cannot be saved. After storage is repaired or T3 restarts, -the durable uncertain request still prevents a second paid start. Remote task and -billing status remain unknown until Kilo confirms the original operation. - -If Kilo explicitly rejects a submitted request while local storage is unavailable, -T3 holds the reservation and keeps that rejection in memory until it can save the -outcome. Reopen history after repairing storage. If T3 exits before that save, the -journal cannot prove the rejection and the request remains uncertain. Stop does -not send a remote interrupt for a known rejected request. +T3 Code supports two Kilo providers: **Kilo** runs the Kilo CLI on the machine +running your environment, and **Kilo Cloud** runs tasks in Kilo's hosted sandboxes. +Add either one in **Settings > Providers**. + +## Local Kilo + +Install Kilo CLI 7.8.3, then add **Kilo** with its binary path and an account +profile. Each profile keeps its own credentials and session history. Sign in with +the official Kilo CLI against that profile; T3 Code never signs in for you. Refresh +the provider after signing in to load your models and agents. + +Changing a profile's credentials ends its running sessions. Start a new thread to +continue with the new account; saved history stays readable. + +Like [OpenCode](./providers-opencode.md), Kilo trusts its native configuration, +plugins, and MCP servers. Only open repositories and profiles whose configuration +you trust. Kilo 7.8.3 also loads `.kilo/mcp.json` and `.kilocode/mcp.json` from the +project. Configured MCP servers can start before any approval, so +[permission modes](./permission-modes.md) govern tool calls, not what Kilo loads. + +Subagents run only with **Full access** in the default interaction mode. Restricted +modes and Plan mode disable them, because Kilo does not pass approval rules on to +child agents reliably. + +## Kilo Cloud + +Add **Kilo Cloud** with a profile directory signed in through the official Kilo +CLI, a GitHub repository Kilo can access, its branch, and a model. Then turn on +**Allow paid cloud execution**. Personal accounts are supported. + +Prompts and the selected repository go to Kilo. T3 Code never uploads local files +or uncommitted changes, and each cloud thread works in its own remote worktree. +Start cloud threads at the project root. Local attachments, terminals, Git +actions, checkpoints, rewind, forks, and generated titles are unavailable in cloud +threads. + +Cloud tasks require **Full access**: Kilo Cloud cannot apply restricted permissions +or Plan mode, so remote shell commands, edits, and subagents can run. Automatic +commits are disabled. Profiles with inherited setup commands, MCP servers, skills, +agents, or environment variables are rejected before a task starts. + +### Costs + +Cloud tasks spend Kilo credit on inference and sandbox time. The thread shows task, +result, sandbox, and billing status separately. A compute estimate can cover a +sandbox shared across your account, so it is not a per-task invoice. Closing T3 Code +does not stop a remote task, and **Stop** does not put the sandbox to sleep. T3 Code +never tops up credit. + +### Stop, reconnect, and results + +**Stop** asks Kilo to interrupt a running task and waits for confirmation. Once Kilo +reports the task complete, T3 Code keeps retrieving its result for up to five +minutes; **Stop** then cancels only that retrieval. If the result does not arrive in +time, the turn fails with a result-retrieval error. Reopening the thread's history +can still retrieve a late result without submitting the task again. + +T3 Code never resends a prompt automatically. If it cannot confirm that Kilo +accepted a prompt, the thread keeps checking Kilo's history for it instead. When +that check cannot finish, the thread pauses and explains why; reopening history +retries the check. Switching accounts does not move or stop existing tasks. diff --git a/knip.jsonc b/knip.jsonc index cb2ab91e03d1..8865bf58869c 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -25,7 +25,6 @@ "scripts/cli.ts", "scripts/evaluate-thread-titles.ts", "scripts/measure-pr-preview.ts", - "scripts/kilo-stream-benchmark.mjs", "scripts/probe-claude-fork-local-rollback-replay.ts", "scripts/update-test-shard-weights.ts", "scripts/verify-background-live.ts", @@ -39,7 +38,6 @@ // Electron loads these bundles by filename rather than importing them. "entry": [ "gnome-extension/extension.js!", - "scripts/kilo-ui-evidence.mjs", "src/electron/WindowsForegroundFocusWorker.ts!", "src/snapShot/GlobalShiftShortcutWorker.ts!", "src/snapShot/RegionSnapShotWorker.ts!", From 43876ccde7d721c1e8de31b09b4ace760d172e75 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 16:10:28 +0000 Subject: [PATCH 29/44] refactor(web): guard the cloud right panel where it renders Keep upstream's rightPanelContent block unchanged and skip it for cloud threads at the two RightPanelTabs call sites instead of re-indenting the whole block. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LguFPcf279M61rSEudqC4q --- apps/web/src/components/ChatView.tsx | 303 +++++++++++++-------------- 1 file changed, 151 insertions(+), 152 deletions(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index ac88fd396dae..364185aa3ee8 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -10581,165 +10581,164 @@ export default function ChatView(props: ChatViewProps) { return ; } - const rightPanelContent = - activeThreadRef && !isCloudThread ? ( - renderedRightPanelSurface?.kind === "preview" ? ( - - { - void onSend(undefined, "auto", "foreground", { annotation, image }); - }} - /> - - ) : renderedRightPanelSurface?.kind === "terminal" ? ( - + { + void onSend(undefined, "auto", "foreground", { annotation, image }); + }} + /> + + ) : renderedRightPanelSurface?.kind === "terminal" ? ( + + ) : renderedRightPanelSurface?.kind === "diff" ? ( + + + + ) : renderedRightPanelSurface?.kind === "pull-request" && !pullRequestsCapabilityKnown ? ( + + ) : renderedRightPanelSurface?.kind === "pull-request" && !supportsPullRequests ? ( + + ) : renderedRightPanelSurface?.kind === "pull-request" ? ( + // No onClose: the surface tab's own X owns closing here, and a second X in the header + // would be the same action twice. The thread context also drops the checkout button, so it + // is only right for the thread's own pull request, whose branch is already under the + // reader's feet. A link the agent wrote can open any other one here, and that one has to be + // checkable out like it is anywhere else. + { + if (activeThreadRef) + useRightPanelStore.getState().openPullRequest(activeThreadRef, { + projectId: reference.projectId, + repository: reference.repository, + number: reference.number, + ...(reference.host ? { host: reference.host } : {}), + }); + }} + threadRef={activeThreadRef} + reference={{ + projectId: renderedRightPanelSurface.projectId as ProjectId, + ...(renderedRightPanelSurface.host ? { host: renderedRightPanelSurface.host } : {}), + repository: renderedRightPanelSurface.repository, + number: renderedRightPanelSurface.number, + }} + context={pullRequestPanelContext( + { + projectId: activeThread.projectId, + pullRequests: visiblePullRequests, + linkedPullRequest: linkedThreadPullRequest, + branchPullRequest: + activeThreadShell?.branchPullRequest ?? activeThread.branchPullRequest, + }, + renderedRightPanelSurface, + )} + composerDraftTarget={composerDraftTarget} + onBack={ + activeThreadRef !== null && pullRequestsSurfaceAvailable && visiblePullRequestCount > 1 + ? addPullRequestsSurface + : undefined + } + /> + ) : renderedRightPanelSurface?.kind === "pull-requests" && activeThreadRef ? ( + + ) : renderedRightPanelSurface?.kind === "device" ? ( + + - ) : renderedRightPanelSurface?.kind === "diff" ? ( - - - - ) : renderedRightPanelSurface?.kind === "pull-request" && !pullRequestsCapabilityKnown ? ( - - ) : renderedRightPanelSurface?.kind === "pull-request" && !supportsPullRequests ? ( - { + closeRightPanelSurface(renderedRightPanelSurface); + useRightPanelStore.getState().show(activeThreadRef); + }} /> - ) : renderedRightPanelSurface?.kind === "pull-request" ? ( - // No onClose: the surface tab's own X owns closing here, and a second X in the header - // would be the same action twice. The thread context also drops the checkout button, so it - // is only right for the thread's own pull request, whose branch is already under the - // reader's feet. A link the agent wrote can open any other one here, and that one has to be - // checkable out like it is anywhere else. - + ) : (renderedRightPanelSurface?.kind === "files" || + renderedRightPanelSurface?.kind === "file") && + ((activeProject && activeWorkspaceRoot) || + (renderedRightPanelSurface.kind === "file" && renderedRightPanelSurface.attachment)) ? ( + + { - if (activeThreadRef) - useRightPanelStore.getState().openPullRequest(activeThreadRef, { - projectId: reference.projectId, - repository: reference.repository, - number: reference.number, - ...(reference.host ? { host: reference.host } : {}), - }); - }} + cwd={activeWorkspaceRoot ?? ""} + projectName={activeProject?.title ?? ""} threadRef={activeThreadRef} - reference={{ - projectId: renderedRightPanelSurface.projectId as ProjectId, - ...(renderedRightPanelSurface.host ? { host: renderedRightPanelSurface.host } : {}), - repository: renderedRightPanelSurface.repository, - number: renderedRightPanelSurface.number, - }} - context={pullRequestPanelContext( - { - projectId: activeThread.projectId, - pullRequests: visiblePullRequests, - linkedPullRequest: linkedThreadPullRequest, - branchPullRequest: - activeThreadShell?.branchPullRequest ?? activeThread.branchPullRequest, - }, - renderedRightPanelSurface, - )} composerDraftTarget={composerDraftTarget} - onBack={ - activeThreadRef !== null && pullRequestsSurfaceAvailable && visiblePullRequestCount > 1 - ? addPullRequestsSurface - : undefined + keybindings={keybindings} + availableEditors={availableEditors} + relativePath={ + renderedRightPanelSurface.kind === "file" + ? renderedRightPanelSurface.relativePath + : null + } + {...(renderedRightPanelSurface.kind === "file" && renderedRightPanelSurface.attachment + ? { attachment: renderedRightPanelSurface.attachment } + : {})} + revealLine={ + renderedRightPanelSurface.kind === "file" + ? (renderedRightPanelSurface.revealLine ?? null) + : null } + revealRequestId={ + renderedRightPanelSurface.kind === "file" + ? renderedRightPanelSurface.revealRequestId + : 0 + } + onOpenFile={openFileSurface} + onPendingChange={handleFilePendingChange} + selectedFilePending={ + renderedRightPanelSurface.kind === "file" && + pendingFileSurfaceIds.has(renderedRightPanelSurface.id) + } + workspaceMutationId={workspaceMutationId} /> - ) : renderedRightPanelSurface?.kind === "pull-requests" && activeThreadRef ? ( - - ) : renderedRightPanelSurface?.kind === "device" ? ( - - { - closeRightPanelSurface(renderedRightPanelSurface); - useRightPanelStore.getState().show(activeThreadRef); - }} - /> - - ) : (renderedRightPanelSurface?.kind === "files" || - renderedRightPanelSurface?.kind === "file") && - ((activeProject && activeWorkspaceRoot) || - (renderedRightPanelSurface.kind === "file" && renderedRightPanelSurface.attachment)) ? ( - - - - ) : null - ) : null; + + ) : null + ) : null; const threadDetailsPanelProps: ThreadDetailsPanelProps = { anchor: threadPanelPopoverAnchorRef, handle: threadPanelPopoverHandle, @@ -11604,7 +11603,7 @@ export default function ChatView(props: ChatViewProps) { pullRequestsAvailable={!isCloudThread && pullRequestsSurfaceAvailable} deviceAvailable={!isCloudThread && activeThreadRef !== null} > - {rightPanelContent} + {isCloudThread ? null : rightPanelContent} ) : null} {rightPanelPresent && shouldUsePlanSidebarSheet && activeThreadRef ? ( @@ -11659,7 +11658,7 @@ export default function ChatView(props: ChatViewProps) { pullRequestsAvailable={!isCloudThread && pullRequestsSurfaceAvailable} deviceAvailable={!isCloudThread && activeThreadRef !== null} > - {rightPanelContent} + {isCloudThread ? null : rightPanelContent} ) : null} From dab8a114476533650aff718af4af2cbfe012067d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 16:28:35 +0000 Subject: [PATCH 30/44] refactor(kilo): share part nodes, request settlement and turn closing Both Kilo adapters built the same part execution nodes, settled runtime requests with their node and item, and closed open records at a turn's end. Move those into helpers beside the existing shared interaction mapping, and express Kilo Cloud capabilities as overrides of local Kilo. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LguFPcf279M61rSEudqC4q --- .../Adapters/KiloAdapterV2.ts | 258 +++++++++--------- .../Adapters/KiloCloudAdapterV2.ts | 175 +++--------- 2 files changed, 171 insertions(+), 262 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts index 568db8452862..b6effad78ed4 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts @@ -45,7 +45,7 @@ const isKiloSessionError = Schema.is(KiloSessionError); export const KILO_PROVIDER = ProviderDriverKind.make("kilo"); -const capabilities: OrchestrationV2ProviderCapabilities = { +export const kiloCapabilities: OrchestrationV2ProviderCapabilities = { sessions: { supportsMultipleProviderThreadsPerSession: false, supportsModelSwitchInSession: true, @@ -314,6 +314,117 @@ export const kiloPermissionReply = ( ? ("always" as const) : ("reject" as const); +/** The execution node for the turn item of an assistant text, reasoning or tool part. */ +export const kiloPartNode = ( + item: Pick< + OrchestrationV2ExecutionNode, + | "threadId" + | "runId" + | "status" + | "providerThreadId" + | "providerTurnId" + | "nativeItemRef" + | "startedAt" + | "completedAt" + > & { readonly nodeId: OrchestrationV2ExecutionNode["id"] }, + rootNodeId: OrchestrationV2ExecutionNode["id"], + kind: "assistant_message" | "reasoning" | "tool_call", +): OrchestrationV2ExecutionNode => ({ + id: item.nodeId, + threadId: item.threadId, + runId: item.runId, + parentNodeId: rootNodeId, + rootNodeId, + kind, + status: item.status, + countsForRun: false, + providerThreadId: item.providerThreadId, + providerTurnId: item.providerTurnId, + nativeItemRef: item.nativeItemRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: item.startedAt, + completedAt: item.completedAt, +}); + +interface KiloRecords { + readonly driver: ProviderDriverKind; + readonly emit: (event: Adapter.ProviderAdapterV2Event) => Effect.Effect; + readonly nodes: ReadonlyMap; + readonly items: ReadonlyMap; +} + +/** Resolves or cancels a runtime request together with the node and item that present it. */ +export const kiloSettleRequest = Effect.fnUntraced(function* ( + records: KiloRecords, + request: OrchestrationV2RuntimeRequest, + turnItemId: OrchestrationV2TurnItem["id"], + outcome: "resolved" | "cancelled", +) { + const at = yield* DateTime.now; + const status = outcome === "resolved" ? "completed" : "interrupted"; + const runtime = { ...request, status: outcome, resolvedAt: at }; + yield* records.emit({ + type: "runtime_request.updated", + driver: records.driver, + runtimeRequest: runtime, + }); + const node = records.nodes.get(request.nodeId); + if (node) + yield* records.emit({ + type: "node.updated", + driver: records.driver, + node: { ...node, status, completedAt: at }, + }); + const item = records.items.get(turnItemId); + if (item) + yield* records.emit({ + type: "turn_item.updated", + driver: records.driver, + turnItem: { ...item, status, completedAt: at, updatedAt: at }, + }); + return runtime; +}); + +/** Ends the nodes, turn items and streaming messages a provider turn left open. */ +export const kiloEndOpenRecords = Effect.fnUntraced(function* ( + records: KiloRecords, + messages: Map, + providerTurnId: OrchestrationV2ProviderTurn["id"], + status: "completed" | "failed" | "interrupted", + at: DateTime.Utc, +) { + const open = (record: OrchestrationV2ExecutionNode | OrchestrationV2TurnItem) => + record.providerTurnId === providerTurnId && + (record.status === "running" || record.status === "waiting"); + for (const node of records.nodes.values()) + if (open(node)) + yield* records.emit({ + type: "node.updated", + driver: records.driver, + node: { ...node, status, completedAt: at }, + }); + for (const item of records.items.values()) + if (open(item)) + yield* records.emit({ + type: "turn_item.updated", + driver: records.driver, + turnItem: { + ...item, + status, + completedAt: at, + updatedAt: at, + ...("streaming" in item ? { streaming: false } : {}), + }, + }); + for (const [id, message] of messages) { + if (!message.streaming) continue; + const completed = { ...message, streaming: false, updatedAt: at }; + messages.set(id, completed); + yield* records.emit({ type: "message.updated", driver: records.driver, message: completed }); + } +}); + export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { readonly instanceId: ProviderInstanceId; readonly continuationKey: string; @@ -328,7 +439,7 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { return Adapter.ProviderAdapterV2.of({ instanceId: options.instanceId, driver: KILO_PROVIDER, - getCapabilities: () => Effect.succeed(capabilities), + getCapabilities: () => Effect.succeed(kiloCapabilities), planSelectionTransition: () => Effect.succeed(turnScopedSelectionTransition()), openSession: Effect.fn("KiloAdapterV2.openSession")(function* (input) { const scope = yield* Effect.scope; @@ -344,7 +455,7 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { cwd: directory, model: input.modelSelection.model, status: "ready" as const, - capabilities, + capabilities: kiloCapabilities, createdAt: now, updatedAt: now, lastError: null, @@ -358,6 +469,7 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { if (event.type === "turn_item.updated") items.set(event.turnItem.id, event.turnItem); return Queue.offer(events, event).pipe(Effect.asVoid); }); + const records = { driver: KILO_PROVIDER, emit, nodes, items }; let ref: KiloSessionRef | undefined; let thread: OrchestrationV2ProviderThread | undefined; let active: @@ -471,42 +583,7 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { runtimeRequest: pending.runtime, }); } - for (const node of nodes.values()) { - if ( - node.providerTurnId !== running.turn.id || - !["running", "waiting"].includes(node.status) - ) - continue; - yield* emit({ - type: "node.updated", - driver: KILO_PROVIDER, - node: { ...node, status, completedAt }, - }); - } - for (const item of items.values()) { - if ( - item.providerTurnId !== running.turn.id || - !["running", "waiting"].includes(item.status) - ) - continue; - yield* emit({ - type: "turn_item.updated", - driver: KILO_PROVIDER, - turnItem: { - ...item, - status, - completedAt, - updatedAt: completedAt, - ...("streaming" in item ? { streaming: false } : {}), - }, - }); - } - for (const [id, message] of messages) { - if (!message.streaming) continue; - const completed = { ...message, streaming: false, updatedAt: completedAt }; - messages.set(id, completed); - yield* emit({ type: "message.updated", driver: KILO_PROVIDER, message: completed }); - } + yield* kiloEndOpenRecords(records, messages, running.turn.id, status, completedAt); for (const [id, child] of subagents) { if (child.runId !== running.input.runId || child.status !== "running") continue; const ended = { ...child, status, completedAt, updatedAt: completedAt }; @@ -650,23 +727,11 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { yield* emit({ type: "node.updated", driver: KILO_PROVIDER, - node: { - id: nodeId, - threadId: running.input.threadId, - runId: running.input.runId, - parentNodeId: running.input.rootNodeId, - rootNodeId: running.input.rootNodeId, - kind: part.type === "text" ? "assistant_message" : "reasoning", - status: base.status, - countsForRun: false, - providerThreadId: thread.id, - providerTurnId: running.turn.id, - nativeItemRef: nativeRef(part.id), - runtimeRequestId: null, - checkpointScopeId: null, - startedAt: base.startedAt, - completedAt: base.completedAt, - }, + node: kiloPartNode( + base, + running.input.rootNodeId, + part.type === "text" ? "assistant_message" : "reasoning", + ), }); yield* emit({ type: "turn_item.updated", @@ -924,23 +989,7 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { yield* emit({ type: "node.updated", driver: KILO_PROVIDER, - node: { - id: nodeId, - threadId: running.input.threadId, - runId: running.input.runId, - parentNodeId: running.input.rootNodeId, - rootNodeId: running.input.rootNodeId, - kind: "tool_call", - status, - countsForRun: false, - providerThreadId: thread.id, - providerTurnId: running.turn.id, - nativeItemRef: nativeRef(part.id), - runtimeRequestId: null, - checkpointScopeId: null, - startedAt: running.turn.startedAt, - completedAt: done ? at : null, - }, + node: kiloPartNode(base, running.input.rootNodeId, "tool_call"), }); yield* emit({ type: "turn_item.updated", @@ -1176,27 +1225,12 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { for (const request of pending) yield* ask(request); for (const saved of requests.values()) { if (saved.runtime.status !== "pending" || present.has(saved.native.id)) continue; - const at = yield* DateTime.now; - saved.runtime = { ...saved.runtime, status: "cancelled", resolvedAt: at }; - yield* emit({ - type: "runtime_request.updated", - driver: KILO_PROVIDER, - runtimeRequest: saved.runtime, - }); - const node = nodes.get(saved.runtime.nodeId); - if (node) - yield* emit({ - type: "node.updated", - driver: KILO_PROVIDER, - node: { ...node, status: "interrupted", completedAt: at }, - }); - const item = items.get(ids.derive.approvalTurnItem({ requestId: saved.runtime.id })); - if (item) - yield* emit({ - type: "turn_item.updated", - driver: KILO_PROVIDER, - turnItem: { ...item, status: "interrupted", completedAt: at, updatedAt: at }, - }); + saved.runtime = yield* kiloSettleRequest( + records, + saved.runtime, + ids.derive.approvalTurnItem({ requestId: saved.runtime.id }), + "cancelled", + ); } if (!active?.admitted) return; const status = yield* wire(client.status(native)); @@ -1627,36 +1661,12 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { ), ); } - const resolved = { - ...pending.runtime, - status: "resolved" as const, - resolvedAt: yield* DateTime.now, - }; - pending.runtime = resolved; - yield* emit({ - type: "runtime_request.updated", - driver: KILO_PROVIDER, - runtimeRequest: resolved, - }); - const node = nodes.get(pending.runtime.nodeId); - if (node) - yield* emit({ - type: "node.updated", - driver: KILO_PROVIDER, - node: { ...node, status: "completed", completedAt: resolved.resolvedAt }, - }); - const item = items.get(ids.derive.approvalTurnItem({ requestId: request.requestId })); - if (item) - yield* emit({ - type: "turn_item.updated", - driver: KILO_PROVIDER, - turnItem: { - ...item, - status: "completed", - completedAt: resolved.resolvedAt, - updatedAt: resolved.resolvedAt, - }, - }); + pending.runtime = yield* kiloSettleRequest( + records, + pending.runtime, + ids.derive.approvalTurnItem({ requestId: request.requestId }), + "resolved", + ); }), ), readThreadSnapshot: (request) => diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts index 3d2f25fa72de..a84638100b79 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -32,97 +32,53 @@ import { makeProviderFailure } from "../ProviderFailure.ts"; import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import { openCodePermissionRules } from "./OpenCodeAdapterV2.ts"; import { openCodeToolTurnItem } from "./OpenCodeToolItems.ts"; -import { kiloInteraction, kiloPermissionReply, kiloQuestionAnswers } from "./KiloAdapterV2.ts"; +import { + kiloCapabilities, + kiloEndOpenRecords, + kiloInteraction, + kiloPartNode, + kiloPermissionReply, + kiloQuestionAnswers, + kiloSettleRequest, +} from "./KiloAdapterV2.ts"; export const KILO_CLOUD_PROVIDER = ProviderDriverKind.make("kilo-cloud"); +// The same native agent runs remotely. T3 cannot reach the sandbox's files, subagents or +// native rewind, and only enforces policy where it answers remote interactions. const capabilities: OrchestrationV2ProviderCapabilities = { - sessions: { - supportsMultipleProviderThreadsPerSession: false, - supportsModelSwitchInSession: true, - supportsProviderSwitchingViaHandoff: false, - supportsRuntimeModeSwitchInSession: false, - pendingRequestsSurviveRestart: true, - }, + ...kiloCapabilities, + sessions: { ...kiloCapabilities.sessions, pendingRequestsSurviveRestart: true }, threads: { - canCreateEmptyThread: true, - canReadThreadSnapshot: true, + ...kiloCapabilities.threads, canRollbackThread: false, canForkThread: false, canForkFromTurn: false, - canForkFromSubagentThread: false, - exposesNativeThreadId: true, - }, - turns: { - exposesNativeTurnId: false, - emitsTurnStarted: true, - emitsTurnCompleted: true, - supportsInterrupt: true, - supportsActiveSteering: false, - supportsSteeringByInterruptRestart: false, - supportsQueuedMessages: false, - terminalStatusQuality: "strong", }, streaming: { + ...kiloCapabilities.streaming, streamsAssistantText: false, streamsReasoning: false, - streamsToolOutput: false, - streamsPlanText: false, - emitsMessageCompleted: true, - }, - tools: { - exposesToolItemIds: true, - emitsToolStarted: true, - emitsToolCompleted: true, - emitsToolOutput: true, - supportsMcpTools: false, - supportsDynamicToolCallbacks: false, }, approvals: { + ...kiloCapabilities.approvals, supportsCommandApproval: false, supportsFileReadApproval: false, supportsFileChangeApproval: false, supportsApplyPatchApproval: false, - approvalsHaveNativeRequestIds: true, - approvalCallbacksAreLiveOnly: true, approvalsCanOriginateFromSubagents: false, }, - planning: { - emitsPlanUpdated: false, - emitsTodoList: false, - emitsProposedPlan: false, - supportsStructuredQuestions: true, - planDeltasHaveItemIds: false, - }, subagents: { + ...kiloCapabilities.subagents, supportsSubagents: false, exposesSubagentThreadIds: false, emitsSubagentLifecycle: false, canWaitForSubagents: false, - canCloseSubagents: false, - canForkSubagentThread: false, - }, - context: { - acceptsSystemContext: false, - acceptsDeveloperContext: false, - acceptsSyntheticUserContext: false, - canGenerateSummaries: false, - canConsumeHandoffSummaries: false, - supportsDeltaHandoff: false, - supportsFullThreadHandoff: false, - maxRecommendedHandoffChars: null, }, checkpointing: { + ...kiloCapabilities.checkpointing, appCanCheckpointFilesystem: false, - supportsNestedCheckpointScopes: false, providerCanRollbackConversation: false, providerRollbackReturnsSnapshot: false, - providerCanReadConversationSnapshot: true, - }, - identity: { - nativeThreadIds: "strong", - nativeTurnIds: "weak", - nativeItemIds: "strong", - nativeRequestIds: "strong", }, runtimePolicy: { enforcement: "client-boundary" }, }; @@ -199,6 +155,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { if (event.type === "turn_item.updated") items.set(event.turnItem.id, event.turnItem); return Queue.offer(queue, event).pipe(Effect.asVoid); }); + const records = { driver, emit, nodes, items }; let thread: OrchestrationV2ProviderThread | undefined; let active: Journal.CloudIntent | undefined; let needsHistoryRestore = false; @@ -230,12 +187,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { const signatures = new Map(); const requests = new Map< RuntimeRequestId, - { - runtime: OrchestrationV2RuntimeRequest; - native: Cloud.CloudInteraction; - node: OrchestrationV2ExecutionNode; - item: OrchestrationV2TurnItem; - } + { runtime: OrchestrationV2RuntimeRequest; native: Cloud.CloudInteraction } >(); const ordinals = new Map(); const ordinal = (id: string) => { @@ -276,18 +228,12 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { entry: NonNullable>, cancelled = false, ) { - const at = yield* DateTime.now; - entry.runtime = { - ...entry.runtime, - status: cancelled ? "cancelled" : "resolved", - resolvedAt: at, - }; - const status = cancelled ? ("interrupted" as const) : ("completed" as const); - entry.node = { ...entry.node, status, completedAt: at }; - entry.item = { ...entry.item, status, completedAt: at, updatedAt: at }; - yield* emit({ type: "runtime_request.updated", driver, runtimeRequest: entry.runtime }); - yield* emit({ type: "node.updated", driver, node: entry.node }); - yield* emit({ type: "turn_item.updated", driver, turnItem: entry.item }); + entry.runtime = yield* kiloSettleRequest( + records, + entry.runtime, + ids.derive.approvalTurnItem({ requestId: entry.runtime.id }), + cancelled ? "cancelled" : "resolved", + ); }); yield* Effect.addFinalizer(() => active ? options.client.forgetAdmission(options.repository, active.messageId) : Effect.void, @@ -338,36 +284,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { if (request.runtime.status !== "pending") continue; yield* resolveRequest(request, true); } - for (const node of nodes.values()) { - if ( - node.providerTurnId === saved.providerTurn.id && - ["running", "waiting"].includes(node.status) - ) - yield* emit({ - type: "node.updated", - driver, - node: { ...node, status: terminal, completedAt: at }, - }); - } - for (const item of items.values()) { - if ( - item.providerTurnId !== saved.providerTurn.id || - !["running", "waiting"].includes(item.status) - ) - continue; - const done = { ...item, status: terminal, completedAt: at, updatedAt: at }; - yield* emit({ - type: "turn_item.updated", - driver, - turnItem: "streaming" in done ? { ...done, streaming: false } : done, - }); - } - for (const [id, message] of messages) { - if (!message.streaming) continue; - const done = { ...message, streaming: false, updatedAt: at }; - messages.set(id, done); - yield* emit({ type: "message.updated", driver, message: done }); - } + yield* kiloEndOpenRecords(records, messages, saved.providerTurn.id, terminal, at); yield* emit( terminal === "failed" ? { @@ -505,28 +422,15 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { yield* emit({ type: "node.updated", driver, - node: { - id: nodeId, - threadId: thread.appThreadId, - runId: correlation.runId, - parentNodeId: correlation.nodeId, - rootNodeId: correlation.nodeId, - kind: - part.type === "tool" - ? "tool_call" - : part.type === "reasoning" - ? "reasoning" - : "assistant_message", - status: base.status, - countsForRun: false, - providerThreadId: thread.id, - providerTurnId: intent.providerTurn.id, - nativeItemRef: base.nativeItemRef, - runtimeRequestId: null, - checkpointScopeId: null, - startedAt: base.startedAt, - completedAt: base.completedAt, - }, + node: kiloPartNode( + base, + correlation.nodeId, + part.type === "tool" + ? "tool_call" + : part.type === "reasoning" + ? "reasoning" + : "assistant_message", + ), }); const item: OrchestrationV2TurnItem = part.type === "tool" @@ -575,12 +479,7 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { ordinal: ordinal(native.id), at: yield* DateTime.now, }); - requests.set(requestId, { - runtime: interaction.runtime, - native, - node: interaction.node, - item: interaction.turnItem, - }); + requests.set(requestId, { runtime: interaction.runtime, native }); yield* emit({ type: "node.updated", driver, node: interaction.node }); yield* emit({ type: "runtime_request.updated", From d3649ace6f616483f1fe53e791f0657ee1bed119 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 16:34:51 +0000 Subject: [PATCH 31/44] test(kilo): share Kilo Cloud fixture setup across cases Every case rebuilt the same credential directory, journal SQLite handle, rejection-write trigger, terminal-event watcher and first-turn restart. Move those into helpers, and park fixture requests through one switch instead of four copies. The cases and their assertions are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LguFPcf279M61rSEudqC4q --- .../Adapters/KiloCloudAdapterV2.test.ts | 551 +++++------------- 1 file changed, 160 insertions(+), 391 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index 846e3339efe6..3a81d0db12f6 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -57,6 +57,7 @@ const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); const encodeIntent = Schema.encodeEffect( Schema.fromJsonString(Schema.toCodecJson(Journal.CloudIntent)), ); +type Parked = "preflight" | "prepare" | "list" | "send"; const fixture = Effect.acquireRelease( Effect.promise(async () => { let submissions = 0; @@ -88,17 +89,15 @@ const fixture = Effect.acquireRelease( signalInterrupt = resolve; }); const control = { + // Parked requests stay open until a test answers them. + park: new Set(), + parked: {} as Partial>, + seen: {} as Partial void>>, prepareStatus: 200, rejectAcceptedPrepare: false, - parkPrepare: false, - parkedPrepare: undefined as NodeHttp.ServerResponse | undefined, - prepareSeen: undefined as (() => void) | undefined, preparePosts: 0, listReads: 0, listStatus: 200, - parkList: false, - parkedList: undefined as NodeHttp.ServerResponse | undefined, - listSeen: undefined as (() => void) | undefined, listClosed: undefined as (() => void) | undefined, sandboxActive: undefined as boolean | undefined, resultReads: 0, @@ -107,9 +106,6 @@ const fixture = Effect.acquireRelease( sessionBranch: "main", malformedSession: false, sendPosts: 0, - parkSend: false, - parkedSend: undefined as NodeHttp.ServerResponse | undefined, - sendSeen: undefined as (() => void) | undefined, profileStatus: 200, personalAccount: true, profileAccount: "fixture-account", @@ -117,9 +113,6 @@ const fixture = Effect.acquireRelease( preflightStatus: 200, malformedPreflight: false, afterBindings: undefined as (() => void) | undefined, - parkedPreflight: undefined as NodeHttp.ServerResponse | undefined, - parkPreflight: false, - preflightSeen: undefined as (() => void) | undefined, status: "completed", requireLocalOverlap: false, dropNextPrepare: false, @@ -155,6 +148,17 @@ const fixture = Effect.acquireRelease( }); request.on("end", () => { const url = new URL(request.url!, "http://localhost"); + const parked = (name: Parked) => { + if (!control.park.has(name)) return false; + control.parked[name] = response; + response.once("close", () => { + if (control.parked[name] === response) delete control.parked[name]; + if (name === "list") control.listClosed?.(); + }); + response.on("error", () => {}); + control.seen[name]?.(); + return true; + }; if (url.pathname.endsWith("/chat/completions")) { localRequests++; localResponse = response; @@ -191,14 +195,7 @@ const fixture = Effect.acquireRelease( return; } if (operation.startsWith("agentProfiles.")) { - if (control.parkPreflight) { - control.parkedPreflight = response; - response.once("close", () => { - if (control.parkedPreflight === response) control.parkedPreflight = undefined; - }); - control.preflightSeen?.(); - return; - } + if (parked("preflight")) return; if (control.preflightStatus !== 200) { response.writeHead(control.preflightStatus); response.end(); @@ -209,15 +206,7 @@ const fixture = Effect.acquireRelease( } if (operation === "cloudAgentNext.prepareSession") { control.preparePosts++; - if (control.parkPrepare) { - control.parkedPrepare = response; - response.once("close", () => { - if (control.parkedPrepare === response) control.parkedPrepare = undefined; - }); - response.on("error", () => {}); - control.prepareSeen?.(); - return; - } + if (parked("prepare")) return; if (control.prepareStatus !== 200) { response.writeHead(control.prepareStatus); response.end(); @@ -247,16 +236,7 @@ const fixture = Effect.acquireRelease( return reply({ cloudAgentSessionId: state.cloud, kiloSessionId: state.native }); } if (operation === "cliSessionsV2.list") control.listReads++; - if (operation === "cliSessionsV2.list" && control.parkList) { - control.parkedList = response; - response.once("close", () => { - if (control.parkedList === response) control.parkedList = undefined; - control.listClosed?.(); - }); - response.on("error", () => {}); - control.listSeen?.(); - return; - } + if (operation === "cliSessionsV2.list" && parked("list")) return; if (operation === "cliSessionsV2.list" && control.listStatus !== 200) { response.writeHead(control.listStatus); response.end(); @@ -317,15 +297,7 @@ const fixture = Effect.acquireRelease( }); if (operation === "cloudAgentNext.sendMessage") { control.sendPosts++; - if (control.parkSend) { - control.parkedSend = response; - response.once("close", () => { - if (control.parkedSend === response) control.parkedSend = undefined; - }); - response.on("error", () => {}); - control.sendSeen?.(); - return; - } + if (parked("send")) return; const payload = input.payload as unknown as { prompt: string }; state.messages.push({ id: input.messageId!, prompt: payload.prompt }); return reply({ @@ -547,6 +519,40 @@ const fixture = Effect.acquireRelease( (fixture) => Effect.promise(fixture.close), ); +const validAuth = '{"kilo":{"type":"api","key":"synthetic"}}'; +/** A scoped data directory holding a valid Kilo credential. */ +const authorizedDirectory = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped(); + yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + yield* fs.writeFileString(`${directory}/data/kilo/auth.json`, validAuth); + return directory; +}); +/** The journal's SQLite file, opened beside the adapter to inject storage faults. */ +const journalDatabase = (directory: string) => + Effect.acquireRelease( + Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), + (db) => Effect.sync(() => db.close()), + ); +const failRejectionWrites = (trigger: string) => + `CREATE TRIGGER ${trigger} BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN SELECT RAISE(FAIL, 'fixture rejection write failure'); END`; +/** The first matching event, read by a consumer forked in the current scope. */ +const firstEvent = Effect.fnUntraced(function* ( + runtime: Adapter.ProviderAdapterV2SessionRuntime, + matches: (event: Adapter.ProviderAdapterV2Event) => boolean, +) { + const found = yield* Deferred.make(); + yield* runtime.events.pipe( + Stream.runForEach((event) => (matches(event) ? Deferred.succeed(found, event) : Effect.void)), + Effect.forkScoped, + ); + return found; +}); +const isTerminal = (event: Adapter.ProviderAdapterV2Event) => event.type === "turn.terminal"; +const awaitingResult = (event: Adapter.ProviderAdapterV2Event) => + event.type === "provider_thread.updated" && + event.providerThread.nativeMetadata?.cloudExecution?.result === "awaiting_result"; + it.live( "completes two isolated cloud threads through SQLite orchestration without touching local workspaces", () => @@ -777,14 +783,17 @@ it.live( assert.equal(new Set(entries.map((entry) => entry.binding?.worktreeId)).size, 2); assert.isTrue(entries.every((entry) => entry.state === "completed")); if (!restore) return yield* Effect.die(new Error("Missing restore input")); + const { threadId, runtimePolicy } = restore; + const openAs = (name: string, cloud = adapter) => + cloud.openSession({ + threadId, + providerSessionId: ProviderSessionId.make(name), + modelSelection, + runtimePolicy, + }); // Simulate loss of the terminal T3 event after the durable journal commit. // Reattaching a fresh runtime must replay terminality without another paid POST. - const restored = yield* adapter.openSession({ - threadId: restore.threadId, - providerSessionId: ProviderSessionId.make("cloud-restored"), - modelSelection, - runtimePolicy: restore.runtimePolicy, - }); + const restored = yield* openAs("cloud-restored"); const restoredThread = yield* restored.ensureThread({ threadId: restore.threadId, existingProviderThread: restore.providerThread, @@ -947,12 +956,7 @@ it.live( // Reopening with paid admission disabled still restores control and native // history; incomplete records from an interrupted turn must stay terminal. const controlOnly = yield* CloudAdapter.make({ ...adapterOptions, allowAdmission: false }); - const reopened = yield* controlOnly.openSession({ - threadId: restore.threadId, - providerSessionId: ProviderSessionId.make("cloud-control-only"), - modelSelection, - runtimePolicy: restore.runtimePolicy, - }); + const reopened = yield* openAs("cloud-control-only", controlOnly); const reopenedThread = yield* reopened.resumeThread({ providerThread: restoredThread }); const snapshot = yield* reopened.readThreadSnapshot({ providerThread: reopenedThread }); assert.isTrue( @@ -994,24 +998,12 @@ it.live( remote.control.omittedItemCount = 0; remote.control.incompleteHistory = false; const retrievalScope = yield* Scope.fork(yield* Effect.scope); - const retrieving = yield* adapter - .openSession({ - threadId: restore.threadId, - providerSessionId: ProviderSessionId.make("retrieval"), - modelSelection, - runtimePolicy: restore.runtimePolicy, - }) - .pipe(Effect.provideService(Scope.Scope, retrievalScope)); + const retrieving = yield* openAs("retrieval").pipe( + Effect.provideService(Scope.Scope, retrievalScope), + ); const retrievalThread = yield* retrieving.resumeThread({ providerThread: restoredThread }); - const awaiting = yield* Deferred.make(); - yield* retrieving.events.pipe( - Stream.runForEach((event) => - event.type === "provider_thread.updated" && - event.providerThread.nativeMetadata?.cloudExecution?.result === "awaiting_result" - ? Deferred.succeed(awaiting, undefined) - : Effect.void, - ), - Effect.forkIn(retrievalScope), + const awaiting = yield* firstEvent(retrieving, awaitingResult).pipe( + Effect.provideService(Scope.Scope, retrievalScope), ); const retrievalInput = { ...restore, @@ -1040,12 +1032,7 @@ it.live( ...waiting, resultRecovery: { ...waiting.resultRecovery!, deadlineMs: 0, nextAttemptMs: 0 }, }); - const afterRestart = yield* adapter.openSession({ - threadId: restore.threadId, - providerSessionId: ProviderSessionId.make("retrieval-restart"), - modelSelection, - runtimePolicy: restore.runtimePolicy, - }); + const afterRestart = yield* openAs("retrieval-restart"); const afterThread = yield* afterRestart.resumeThread({ providerThread: retrievalThread }); const retrievalEvents: Adapter.ProviderAdapterV2Event[] = []; const retrievalFailed = yield* Deferred.make(); @@ -1101,23 +1088,9 @@ it.live( assert.equal(remote.submissions(), 2); // Stop during retrieval is local cancellation, never a remote interrupt. remote.control.missingHistory = true; - const cancelling = yield* adapter.openSession({ - threadId: restore.threadId, - providerSessionId: ProviderSessionId.make("retrieval-cancel"), - modelSelection, - runtimePolicy: restore.runtimePolicy, - }); + const cancelling = yield* openAs("retrieval-cancel"); const cancelThread = yield* cancelling.resumeThread({ providerThread: afterThread }); - const cancelAwaiting = yield* Deferred.make(); - yield* cancelling.events.pipe( - Stream.runForEach((event) => - event.type === "provider_thread.updated" && - event.providerThread.nativeMetadata?.cloudExecution?.result === "awaiting_result" - ? Deferred.succeed(cancelAwaiting, undefined) - : Effect.void, - ), - Effect.forkScoped, - ); + const cancelAwaiting = yield* firstEvent(cancelling, awaitingResult); yield* cancelling.startTurn({ ...retrievalInput, providerThread: cancelThread, @@ -1154,14 +1127,9 @@ it.live( remote.control.historyMode = mode; remote.control.historyReads = 0; const testScope = yield* Scope.fork(yield* Effect.scope); - const runtime = yield* adapter - .openSession({ - threadId: restore.threadId, - providerSessionId: ProviderSessionId.make(`result-${mode}`), - modelSelection, - runtimePolicy: restore.runtimePolicy, - }) - .pipe(Effect.provideService(Scope.Scope, testScope)); + const runtime = yield* openAs(`result-${mode}`).pipe( + Effect.provideService(Scope.Scope, testScope), + ); const selected = yield* runtime.resumeThread({ providerThread: cancelThread }); const done = yield* Deferred.make(); const waitingResult = yield* Deferred.make(); @@ -1364,7 +1332,29 @@ const admissionHarness = Effect.fn("admissionHarness")(function* ( modelSelection, runtimePolicy, }); - return { open, turn, journal, client }; + /** Starts a first turn in a runtime that then closes, as an app restart would. */ + const startThenClose = Effect.fnUntraced(function* ( + until?: (event: Adapter.ProviderAdapterV2Event) => boolean, + ) { + const scope = yield* Scope.fork(yield* Effect.scope); + yield* Effect.gen(function* () { + const first = yield* open; + const seen = until ? yield* firstEvent(first.runtime, until) : undefined; + yield* first.runtime.startTurn(turn(first.thread)); + if (seen) yield* Deferred.await(seen); + }).pipe(Effect.provideService(Scope.Scope, scope)); + yield* Scope.close(scope, Exit.void); + }); + return { open, turn, journal, client, startThenClose }; +}); + +const uncertainAdmission = Effect.gen(function* () { + const remote = yield* fixture; + remote.control.dropNextPrepare = true; + remote.control.hideAdmissions = true; + const directory = yield* authorizedDirectory; + yield* (yield* admissionHarness(remote, directory)).startThenClose(); + return { remote, directory }; }); it.live.each([ @@ -1383,11 +1373,9 @@ it.live.each([ Effect.gen(function* () { const remote = yield* fixture; const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); + const directory = yield* authorizedDirectory; const auth = `${directory}/data/kilo/auth.json`; - const validAuth = '{"kilo":{"type":"api","key":"synthetic-test-token"}}'; - yield* fs.writeFileString(auth, mode === "credential" ? "{}" : validAuth); + if (mode === "credential") yield* fs.writeFileString(auth, "{}"); if (mode === "404" || mode === "503") remote.control.preflightStatus = Number(mode); if (mode === "malformed") remote.control.malformedPreflight = true; if (mode === "profile-503") remote.control.profileStatus = 503; @@ -1422,13 +1410,7 @@ it.live.each([ const restarted = yield* admissionHarness(remote, directory); const next = yield* restarted.open; remote.control.status = "failed"; - const done = yield* Deferred.make(); - yield* next.runtime.events.pipe( - Stream.runForEach((event) => - event.type === "turn.terminal" ? Deferred.succeed(done, undefined) : Effect.void, - ), - Effect.forkScoped, - ); + const done = yield* firstEvent(next.runtime, isTerminal); yield* next.runtime.startTurn(restarted.turn(next.thread, 2)); yield* Deferred.await(done); assert.equal(remote.submissions(), 1); @@ -1442,21 +1424,8 @@ it.live.each([false, true])( "keeps a lost POST uncertain across restart and Stop, legacy=%s", (legacy) => Effect.gen(function* () { - const remote = yield* fixture; - remote.control.dropNextPrepare = true; - remote.control.hideAdmissions = true; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); + const { remote, directory } = yield* uncertainAdmission; const harness = yield* admissionHarness(remote, directory); - const scope = yield* Scope.fork(yield* Effect.scope); - const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, scope)); - yield* first.runtime.startTurn(harness.turn(first.thread)); - yield* Scope.close(scope, Exit.void); let uncertain = (yield* harness.journal.read)[0]!; assert.equal(uncertain.state, "admission_unknown"); assert.equal(uncertain.submissionPhase, "post_attempted"); @@ -1492,16 +1461,10 @@ it.live( () => Effect.gen(function* () { const remote = yield* fixture; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); + const directory = yield* authorizedDirectory; const arrived = yield* Deferred.make(); - remote.control.parkPreflight = true; - remote.control.preflightSeen = () => Deferred.doneUnsafe(arrived, Effect.void); + remote.control.park.add("preflight"); + remote.control.seen.preflight = () => Deferred.doneUnsafe(arrived, Effect.void); const firstHarness = yield* admissionHarness(remote, directory); const first = yield* firstHarness.open; const pending = yield* first.runtime @@ -1517,9 +1480,9 @@ it.live( providerThread: reopened.thread, providerTurnId: recovered.providerTurn.id, }); - remote.control.parkPreflight = false; - remote.control.parkedPreflight!.writeHead(200, { "content-type": "application/json" }); - remote.control.parkedPreflight!.end('{"result":{"data":[]}}'); + remote.control.park.delete("preflight"); + remote.control.parked.preflight!.writeHead(200, { "content-type": "application/json" }); + remote.control.parked.preflight!.end('{"result":{"data":[]}}'); const originalExit = yield* Fiber.await(pending); assert.isTrue(Exit.isFailure(originalExit)); // stale CAS cannot revive the reservation assert.equal(remote.submissions(), 0); @@ -1534,29 +1497,9 @@ it.live( Effect.gen(function* () { const remote = yield* fixture; remote.control.missingHistory = true; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); + const directory = yield* authorizedDirectory; const harness = yield* admissionHarness(remote, directory); - const firstScope = yield* Scope.fork(yield* Effect.scope); - const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, firstScope)); - const waiting = yield* Deferred.make(); - yield* first.runtime.events.pipe( - Stream.runForEach((event) => - event.type === "provider_thread.updated" && - event.providerThread.nativeMetadata?.cloudExecution?.result === "awaiting_result" - ? Deferred.succeed(waiting, undefined) - : Effect.void, - ), - Effect.forkIn(firstScope), - ); - yield* first.runtime.startTurn(harness.turn(first.thread)); - yield* Deferred.await(waiting); - yield* Scope.close(firstScope, Exit.void); + yield* harness.startThenClose(awaitingResult); const saved = (yield* harness.journal.read)[0]!; assert.equal(saved.state, "awaiting_result"); assert.equal(saved.resultRecovery?.attempts, 1); @@ -1594,13 +1537,7 @@ it.live( // Reattach the original turn at expiry. Its terminal event must be replayable, // with remote completion retained and no replacement paid submission. - const terminal = yield* Deferred.make(); - yield* next.runtime.events.pipe( - Stream.runForEach((event) => - event.type === "turn.terminal" ? Deferred.succeed(terminal, event) : Effect.void, - ), - Effect.forkScoped, - ); + const terminal = yield* firstEvent(next.runtime, isTerminal); yield* next.runtime .startTurn({ ...restarted.turn(next.thread), reattach: true }) .pipe(Effect.provideService(Clock.Clock, clockAt(baseClock, deadline + 1))); @@ -1612,13 +1549,7 @@ it.live( assert.equal(ended.resultStatus, "unavailable"); assert.equal(remote.submissions(), 1); const replay = yield* restarted.open; - const replayed = yield* Deferred.make(); - yield* replay.runtime.events.pipe( - Stream.runForEach((event) => - event.type === "turn.terminal" ? Deferred.succeed(replayed, event) : Effect.void, - ), - Effect.forkScoped, - ); + const replayed = yield* firstEvent(replay.runtime, isTerminal); yield* replay.runtime.startTurn({ ...restarted.turn(replay.thread), reattach: true }); const event = yield* Deferred.await(replayed); assert.isTrue(event.type === "turn.terminal" && event.status === "failed"); @@ -1642,18 +1573,7 @@ it.live.each(["404", "503", "malformed", "credential"] as const)( const validAuth = '{"kilo":{"type":"api","key":"synthetic"}}'; yield* fs.writeFileString(auth, validAuth); const harness = yield* admissionHarness(remote, directory); - const firstScope = yield* Scope.fork(yield* Effect.scope); - const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, firstScope)); - const done = yield* Deferred.make(); - yield* first.runtime.events.pipe( - Stream.runForEach((event) => - event.type === "turn.terminal" ? Deferred.succeed(done, undefined) : Effect.void, - ), - Effect.forkIn(firstScope), - ); - yield* first.runtime.startTurn(harness.turn(first.thread)); - yield* Deferred.await(done); - yield* Scope.close(firstScope, Exit.void); + yield* harness.startThenClose(isTerminal); const binding = (yield* harness.journal.read)[0]!.binding; assert.isNotNull(binding); if (mode === "404" || mode === "503") remote.control.sessionStatus = Number(mode); @@ -1678,13 +1598,7 @@ it.live.each(["404", "503", "malformed", "credential"] as const)( yield* fs.writeFileString(auth, validAuth); const restarted = yield* admissionHarness(remote, directory); const third = yield* restarted.open; - const completed = yield* Deferred.make(); - yield* third.runtime.events.pipe( - Stream.runForEach((event) => - event.type === "turn.terminal" ? Deferred.succeed(completed, undefined) : Effect.void, - ), - Effect.forkScoped, - ); + const completed = yield* firstEvent(third.runtime, isTerminal); yield* third.runtime.startTurn(restarted.turn(third.thread, 3)); yield* Deferred.await(completed); assert.equal(remote.submissions(), 1); @@ -1701,18 +1615,8 @@ it.live( const remote = yield* fixture; remote.control.dropNextPrepare = true; remote.control.listStatus = 503; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); - const harness = yield* admissionHarness(remote, directory); - const scope = yield* Scope.fork(yield* Effect.scope); - const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, scope)); - yield* first.runtime.startTurn(harness.turn(first.thread)); - yield* Scope.close(scope, Exit.void); + const directory = yield* authorizedDirectory; + yield* (yield* admissionHarness(remote, directory)).startThenClose(); const restarted = yield* admissionHarness(remote, directory); const next = yield* restarted.open; const baseClock = yield* Clock.Clock; @@ -1737,13 +1641,7 @@ it.live( remote.control.resultReads = 0; remote.control.status = "running"; // Inference can complete while sandbox and billing remain active. remote.control.completeAfterResultReads = 2; - const done = yield* Deferred.make(); - yield* recovered.runtime.events.pipe( - Stream.runForEach((event) => - event.type === "turn.terminal" ? Deferred.succeed(done, undefined) : Effect.void, - ), - Effect.forkScoped, - ); + const done = yield* firstEvent(recovered.runtime, isTerminal); yield* recovered.runtime.readThreadSnapshot({ providerThread: recovered.thread }); yield* Deferred.await(done); // watcher must finish without a second snapshot request const complete = (yield* again.journal.read)[0]!; @@ -1768,28 +1666,14 @@ it.effect( "bounds stalled admission-list reads across durable failures without releasing a paid intent", () => Effect.gen(function* () { - const remote = yield* fixture; - remote.control.dropNextPrepare = true; - remote.control.hideAdmissions = true; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); - const harness = yield* admissionHarness(remote, directory); - const firstScope = yield* Scope.fork(yield* Effect.scope); - const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, firstScope)); - yield* first.runtime.startTurn(harness.turn(first.thread)); - yield* Scope.close(firstScope, Exit.void); + const { remote, directory } = yield* uncertainAdmission; const restarted = yield* admissionHarness(remote, directory); const next = yield* restarted.open; - remote.control.parkList = true; + remote.control.park.add("list"); for (let attempt = 1; attempt <= 3; attempt++) { const requestSeen = yield* Deferred.make(); const requestClosed = yield* Deferred.make(); - remote.control.listSeen = () => Deferred.doneUnsafe(requestSeen, Effect.void); + remote.control.seen.list = () => Deferred.doneUnsafe(requestSeen, Effect.void); remote.control.listClosed = () => Deferred.doneUnsafe(requestClosed, Effect.void); const reading = yield* next.runtime .readThreadSnapshot({ providerThread: next.thread }) @@ -1809,25 +1693,6 @@ it.effect( 20_000, ); -const uncertainAdmission = Effect.gen(function* () { - const remote = yield* fixture; - remote.control.dropNextPrepare = true; - remote.control.hideAdmissions = true; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); - const harness = yield* admissionHarness(remote, directory); - const firstScope = yield* Scope.fork(yield* Effect.scope); - const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, firstScope)); - yield* first.runtime.startTurn(harness.turn(first.thread)); - yield* Scope.close(firstScope, Exit.void); - return { remote, directory }; -}); - it.live( "bounds recovery when real SQLite UPDATEs fail and retains the paid reservation", () => @@ -1836,10 +1701,7 @@ it.live( remote.control.listStatus = 503; const harness = yield* admissionHarness(remote, directory); const opened = yield* harness.open; - const db = yield* Effect.acquireRelease( - Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), - (db) => Effect.sync(() => db.close()), - ); + const db = yield* journalDatabase(directory); db.exec( "CREATE TRIGGER fail_recovery_save BEFORE UPDATE ON intents BEGIN SELECT RAISE(FAIL, 'fixture write failure'); END", ); @@ -1959,22 +1821,10 @@ it.live( Effect.gen(function* () { const remote = yield* fixture; remote.control.prepareStatus = 400; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); + const directory = yield* authorizedDirectory; const harness = yield* admissionHarness(remote, directory); const opened = yield* harness.open; - const terminal = yield* Deferred.make(); - yield* opened.runtime.events.pipe( - Stream.runForEach((event) => - event.type === "turn.terminal" ? Deferred.succeed(terminal, event) : Effect.void, - ), - Effect.forkScoped, - ); + const terminal = yield* firstEvent(opened.runtime, isTerminal); yield* opened.runtime.startTurn(harness.turn(opened.thread)); const event = yield* Deferred.await(terminal); assert.equal(event.type, "turn.terminal"); @@ -2028,10 +1878,7 @@ it.live( const saved = (yield* harness.journal.read)[0]!; // A concurrent preflight owner may finish Stop before this reader refreshes. // SQL restores that durable boundary without loosening production phase guards. - const db = yield* Effect.acquireRelease( - Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), - (db) => Effect.sync(() => db.close()), - ); + const db = yield* journalDatabase(directory); db.prepare("UPDATE intents SET state = ?, body = ? WHERE operation_key = ?").run( "interrupted", yield* encodeIntent({ @@ -2064,17 +1911,14 @@ it.live.each(["pause", "terminal"] as const)( admissionRecoveryFailures: outcome === "terminal" ? 2 : 0, }); const opened = yield* harness.open; - const db = yield* Effect.acquireRelease( - Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), - (db) => Effect.sync(() => db.close()), - ); + const db = yield* journalDatabase(directory); if (outcome === "terminal") db.exec( "CREATE TRIGGER reject_pause BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.admissionRecoveryPaused') = 1 BEGIN SELECT RAISE(FAIL, 'fixture pause write failure'); END", ); - remote.control.parkList = true; + remote.control.park.add("list"); const seen = yield* Deferred.make(); - remote.control.listSeen = () => Deferred.doneUnsafe(seen, Effect.void); + remote.control.seen.list = () => Deferred.doneUnsafe(seen, Effect.void); const reading = yield* opened.runtime .readThreadSnapshot({ providerThread: opened.thread }) .pipe(Effect.forkScoped); @@ -2094,9 +1938,9 @@ it.live.each(["pause", "terminal"] as const)( }, }, ); - remote.control.parkList = false; - remote.control.parkedList!.writeHead(503); - remote.control.parkedList!.end(); + remote.control.park.delete("list"); + remote.control.parked.list!.writeHead(503); + remote.control.parked.list!.end(); yield* Fiber.join(reading); const final = (yield* harness.journal.read)[0]!; if (outcome === "pause") { @@ -2119,26 +1963,14 @@ it.live.each(["storage", "revision"] as const)( (failure) => Effect.gen(function* () { const remote = yield* fixture; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); + const directory = yield* authorizedDirectory; const harness = yield* admissionHarness(remote, directory); const opened = yield* harness.open; - const db = yield* Effect.acquireRelease( - Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), - (db) => Effect.sync(() => db.close()), - ); - if (failure === "storage") - db.exec( - "CREATE TRIGGER reject_outcome BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN SELECT RAISE(FAIL, 'fixture rejection write failure'); END", - ); - remote.control.parkPrepare = true; + const db = yield* journalDatabase(directory); + if (failure === "storage") db.exec(failRejectionWrites("reject_outcome")); + remote.control.park.add("prepare"); const seen = yield* Deferred.make(); - remote.control.prepareSeen = () => Deferred.doneUnsafe(seen, Effect.void); + remote.control.seen.prepare = () => Deferred.doneUnsafe(seen, Effect.void); const starting = yield* opened.runtime .startTurn(harness.turn(opened.thread)) .pipe(Effect.forkScoped); @@ -2147,8 +1979,8 @@ it.live.each(["storage", "revision"] as const)( const other = yield* Journal.make(`${directory}/journal`); yield* other.save({ ...(yield* other.read)[0]!, admissionRecoveryFailures: 1 }); } - remote.control.parkedPrepare!.writeHead(400); - remote.control.parkedPrepare!.end(); + remote.control.parked.prepare!.writeHead(400); + remote.control.parked.prepare!.end(); yield* Fiber.join(starting); assert.equal(remote.control.preparePosts, 1); assert.equal(remote.control.listReads, 0); @@ -2172,7 +2004,7 @@ it.live.each(["storage", "revision"] as const)( assert.equal(saved.state, "failed"); assert.isTrue(saved.submissionRejected); assert.equal(remote.control.listReads, 0); - remote.control.parkPrepare = false; + remote.control.park.delete("prepare"); yield* opened.runtime.startTurn(harness.turn(opened.thread, 2)); assert.equal(remote.control.preparePosts, 2); }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), @@ -2206,51 +2038,24 @@ it.live.each(["accepted", "storage", "storage-accepted", "storage-interrupted"] Effect.gen(function* () { const remote = yield* fixture; remote.control.status = "failed"; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); + const directory = yield* authorizedDirectory; const harness = yield* admissionHarness(remote, directory); - const firstScope = yield* Scope.fork(yield* Effect.scope); - const first = yield* harness.open.pipe(Effect.provideService(Scope.Scope, firstScope)); - const done = yield* Deferred.make(); - yield* first.runtime.events.pipe( - Stream.runForEach((event) => - event.type === "turn.terminal" ? Deferred.succeed(done, undefined) : Effect.void, - ), - Effect.forkIn(firstScope), - ); - yield* first.runtime.startTurn(harness.turn(first.thread)); - yield* Deferred.await(done); - yield* Scope.close(firstScope, Exit.void); + yield* harness.startThenClose(isTerminal); const originalBinding = (yield* harness.journal.read)[0]!.binding; remote.control.status = "running"; - remote.control.parkSend = true; + remote.control.park.add("send"); const second = yield* harness.open; - const monitored = yield* Deferred.make(); - yield* second.runtime.events.pipe( - Stream.runForEach((event) => + const monitored = yield* firstEvent( + second.runtime, + (event) => event.type === "provider_thread.updated" && event.providerThread.nativeMetadata?.cloudExecution?.sandbox === "active" && - event.providerThread.nativeMetadata.cloudExecution.billing === "active" - ? Deferred.succeed(monitored, undefined) - : Effect.void, - ), - Effect.forkScoped, + event.providerThread.nativeMetadata.cloudExecution.billing === "active", ); - const db = yield* Effect.acquireRelease( - Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), - (db) => Effect.sync(() => db.close()), - ); - if (mode !== "accepted") - db.exec( - "CREATE TRIGGER reject_followup BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN SELECT RAISE(FAIL, 'fixture rejection write failure'); END", - ); + const db = yield* journalDatabase(directory); + if (mode !== "accepted") db.exec(failRejectionWrites("reject_followup")); const seen = yield* Deferred.make(); - remote.control.sendSeen = () => Deferred.doneUnsafe(seen, Effect.void); + remote.control.seen.send = () => Deferred.doneUnsafe(seen, Effect.void); const starting = yield* second.runtime .startTurn(harness.turn(second.thread, 2)) .pipe(Effect.forkScoped); @@ -2258,8 +2063,8 @@ it.live.each(["accepted", "storage", "storage-accepted", "storage-interrupted"] const other = yield* Journal.make(`${directory}/journal`); if (mode === "accepted") yield* other.save({ ...(yield* other.read)[1]!, remoteState: "running" }); - remote.control.parkedSend!.writeHead(400); - remote.control.parkedSend!.end(); + remote.control.parked.send!.writeHead(400); + remote.control.parked.send!.end(); yield* Fiber.join(starting); const saved = (yield* other.read)[1]!; assert.equal(saved.state, "admission_unknown"); @@ -2307,13 +2112,7 @@ it.effect.each(["start", "stop"] as const)( Effect.gen(function* () { const remote = yield* fixture; remote.control.prepareStatus = 400; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); + const directory = yield* authorizedDirectory; const committed = yield* Deferred.make(); const release = yield* Deferred.make(); let park = mode === "start"; @@ -2333,14 +2132,9 @@ it.effect.each(["start", "stop"] as const)( ), })); const opened = yield* harness.open; - const db = yield* Effect.acquireRelease( - Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), - (db) => Effect.sync(() => db.close()), - ); + const db = yield* journalDatabase(directory); if (mode === "stop") { - db.exec( - "CREATE TRIGGER reject_outcome BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN SELECT RAISE(FAIL, 'fixture rejection write failure'); END", - ); + db.exec(failRejectionWrites("reject_outcome")); yield* opened.runtime.startTurn(harness.turn(opened.thread)); db.exec("DROP TRIGGER reject_outcome"); park = true; @@ -2393,23 +2187,12 @@ it.live.each(["start", "stop"] as const)( Effect.gen(function* () { const remote = yield* fixture; remote.control.prepareStatus = 400; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); + const directory = yield* authorizedDirectory; const harness = yield* admissionHarness(remote, directory); const opened = yield* harness.open; - const db = yield* Effect.acquireRelease( - Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), - (db) => Effect.sync(() => db.close()), - ); + const db = yield* journalDatabase(directory); if (mode === "stop") { - db.exec( - "CREATE TRIGGER reject_outcome BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN SELECT RAISE(FAIL, 'fixture rejection write failure'); END", - ); + db.exec(failRejectionWrites("reject_outcome")); yield* opened.runtime.startTurn(harness.turn(opened.thread)); db.exec("DROP TRIGGER reject_outcome"); } @@ -2440,10 +2223,7 @@ it.live.each(["resume", "read-failure"] as const)( remote.control.listStatus = 503; const other = yield* Journal.make(`${directory}/journal`); yield* other.save({ ...(yield* other.read)[0]!, admissionRecoveryFailures: 2 }); - const db = yield* Effect.acquireRelease( - Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), - (db) => Effect.sync(() => db.close()), - ); + const db = yield* journalDatabase(directory); let hidden = false; let raced = false; const harness = yield* admissionHarness(remote, directory, (journal) => ({ @@ -2512,22 +2292,11 @@ it.effect.each(["terminal-binding", "prepared", "prepared-write-failure"] as con Effect.gen(function* () { const remote = yield* fixture; remote.control.rejectAcceptedPrepare = true; - const fs = yield* FileSystem.FileSystem; - const directory = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(`${directory}/data/kilo`, { recursive: true }); - yield* fs.writeFileString( - `${directory}/data/kilo/auth.json`, - '{"kilo":{"type":"api","key":"synthetic"}}', - ); + const directory = yield* authorizedDirectory; const harness = yield* admissionHarness(remote, directory); const opened = yield* harness.open; - const db = yield* Effect.acquireRelease( - Effect.sync(() => new NodeSqlite.DatabaseSync(`${directory}/journal/intents.sqlite`)), - (db) => Effect.sync(() => db.close()), - ); - db.exec( - "CREATE TRIGGER reject_outcome BEFORE UPDATE ON intents WHEN json_extract(NEW.body, '$.submissionRejected') = 1 BEGIN SELECT RAISE(FAIL, 'fixture rejection write failure'); END", - ); + const db = yield* journalDatabase(directory); + db.exec(failRejectionWrites("reject_outcome")); yield* opened.runtime.startTurn(harness.turn(opened.thread)); const saved = (yield* harness.journal.read)[0]!; const found = yield* harness.client.findAdmission("fixture/repo", saved.messageId); From 5050de8067b79380f9ed38581a170923cfd12cc3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 16:40:35 +0000 Subject: [PATCH 32/44] refactor(orchestration): read remote execution from capabilities RunExecutionService already reads appCanCheckpointFilesystem from the session. Use it, and the generic reattach flag, instead of naming the kilo-cloud driver. Keep the RunExecutionService tests at upstream's indentation and share their local-workspace capability mock. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LguFPcf279M61rSEudqC4q --- .../RunExecutionService.test.ts | 513 +++++++++--------- .../orchestration-v2/RunExecutionService.ts | 21 +- 2 files changed, 254 insertions(+), 280 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 8b92a55378aa..89f0b520e0e5 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -55,6 +55,10 @@ import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import * as RunExecutionService from "./RunExecutionService.ts"; import * as RunFinalizationService from "./RunFinalizationService.ts"; +// Sessions whose provider runs in the local workspace, which T3 checkpoints. +const localWorkspaceSession = { + capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, +}; const driver = ProviderDriverKind.make("codex"); const RunExecutionTestLayer = RunExecutionService.layer.pipe( @@ -545,7 +549,7 @@ it.effect("rechecks run ownership immediately before calling the provider", () = providerTurnId: null, } as OrchestrationV2RunAttempt; const session = { - providerSession: { capabilities: { checkpointing: { appCanCheckpointFilesystem: true } } }, + providerSession: localWorkspaceSession, events: Stream.never, startTurn: () => Ref.update(providerStarts, (count) => count + 1), } as unknown as ProviderAdapterV2SessionRuntime; @@ -592,7 +596,7 @@ it.effect("rechecks run ownership immediately before calling the provider", () = }).pipe(Effect.provide(RunExecutionTestLayer)), ); -it.effect.each([ +const readFailures = [ { driverName: "codex", reattach: false, failAt: 1 }, { driverName: "kilo", reattach: false, failAt: 0 }, { driverName: "kilo", reattach: false, failAt: 1 }, @@ -600,140 +604,140 @@ it.effect.each([ { driverName: "kilo-cloud", reattach: false, failAt: 1 }, { driverName: "kilo-cloud", reattach: true, failAt: 0 }, { driverName: "kilo-cloud", reattach: true, failAt: 1 }, -])( - "fences ownership-read failure for $driverName reattach=$reattach at check $failAt", - ({ driverName, reattach, failAt }) => - Effect.gen(function* () { - const driver = ProviderDriverKind.make(driverName); - const checkpointFilesystem = driverName !== "kilo-cloud"; - const baselineCalls = yield* Ref.make(0); - const closedSubscriptions = yield* Ref.make(0); - const guardCalls = yield* Ref.make(0); - const providerStarts = yield* Ref.make(0); - const writes = yield* Ref.make>([]); - const threadId = ThreadId.make("thread:run-execution-start-guard-read"); - const runId = RunId.make("run:run-execution-start-guard-read"); - const attemptId = RunAttemptId.make("attempt:run-execution-start-guard-read"); - const providerInstanceId = ProviderInstanceId.make(driverName); - const testLayer = RunExecutionService.layer.pipe( - Layer.provide( - Layer.mergeAll( - Layer.mock(CheckpointService.CheckpointServiceV2)({ - captureBaseline: () => Ref.update(baselineCalls, (n) => n + 1), - }), - Layer.mock(EventSink.EventSinkV2)({ - writeIfRunCurrent: (input) => { - assert.equal(input.runId, runId); - assert.equal(input.activeAttemptId, attemptId); - assert.equal(input.expectedStatus, "running"); - return Ref.update(writes, (current) => [...current, ...input.events]).pipe( - Effect.as({ committed: true, storedEvents: [] }), - ); - }, - }), - IdAllocator.layer, - Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ - ingestNormalized: () => Effect.succeed([]), - }), - ServerSettings.layerTest(), - ), +]; +it.effect.each(readFailures)("ownership read failure $driverName/$reattach/$failAt", (input) => + Effect.gen(function* () { + const { driverName, reattach, failAt } = input; + const driver = ProviderDriverKind.make(driverName); + const checkpointFilesystem = driverName !== "kilo-cloud"; + const baselineCalls = yield* Ref.make(0); + const closedSubscriptions = yield* Ref.make(0); + const guardCalls = yield* Ref.make(0); + const providerStarts = yield* Ref.make(0); + const writes = yield* Ref.make>([]); + const threadId = ThreadId.make("thread:run-execution-start-guard-read"); + const runId = RunId.make("run:run-execution-start-guard-read"); + const attemptId = RunAttemptId.make("attempt:run-execution-start-guard-read"); + const providerInstanceId = ProviderInstanceId.make(driverName); + const testLayer = RunExecutionService.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointService.CheckpointServiceV2)({ + captureBaseline: () => Ref.update(baselineCalls, (n) => n + 1), + }), + Layer.mock(EventSink.EventSinkV2)({ + writeIfRunCurrent: (input) => { + assert.equal(input.runId, runId); + assert.equal(input.activeAttemptId, attemptId); + assert.equal(input.expectedStatus, "running"); + return Ref.update(writes, (current) => [...current, ...input.events]).pipe( + Effect.as({ committed: true, storedEvents: [] }), + ); + }, + }), + IdAllocator.layer, + Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + ingestNormalized: () => Effect.succeed([]), + }), + ServerSettings.layerTest(), ), - ); + ), + ); - yield* Effect.gen(function* () { - const runExecution = yield* RunExecutionService.RunExecutionServiceV2; - const result = yield* Effect.exit( - runExecution.startRootRun({ - commandId: CommandId.make("command:run-execution-start-guard-read"), - appThread: { id: threadId } as OrchestrationV2AppThread, - providerSessionId: ProviderSessionId.make("session:run-execution-start-guard-read"), - reattach, - session: { - driver, - providerSession: { - capabilities: { - checkpointing: { appCanCheckpointFilesystem: checkpointFilesystem }, - }, + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionService.RunExecutionServiceV2; + const result = yield* Effect.exit( + runExecution.startRootRun({ + commandId: CommandId.make("command:run-execution-start-guard-read"), + appThread: { id: threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make("session:run-execution-start-guard-read"), + reattach, + session: { + driver, + providerSession: { + capabilities: { + checkpointing: { appCanCheckpointFilesystem: checkpointFilesystem }, }, - subscribeEvents: Effect.succeed({ - events: Stream.never, - close: Ref.update(closedSubscriptions, (n) => n + 1), - }), + }, + subscribeEvents: Effect.succeed({ events: Stream.never, - startTurn: () => Ref.update(providerStarts, (count) => count + 1), - } as unknown as ProviderAdapterV2SessionRuntime, - run: { id: runId, threadId, ordinal: 1, providerInstanceId } as OrchestrationV2Run, - rootNode: { - id: NodeId.make("node:run-execution-start-guard-read"), - } as OrchestrationV2ExecutionNode, - checkpointScope: { - id: CheckpointScopeId.make("checkpoint-scope:run-execution-start-guard-read"), - } as OrchestrationV2CheckpointScope, - providerThread: { - id: ProviderThreadId.make("provider-thread:run-execution-start-guard-read"), - driver, - } as OrchestrationV2ProviderThread, - attempt: { id: attemptId, providerTurnId: null } as OrchestrationV2RunAttempt, - attemptId, - providerTurnOrdinal: 1, - // The preparation check passes; the check right before the provider - // call cannot read the run. - shouldStartProviderTurn: () => - Ref.getAndUpdate(guardCalls, (calls) => calls + 1).pipe( - Effect.flatMap((calls) => - calls < failAt - ? Effect.succeed(true) - : Effect.fail( - new ProjectionStore.ProjectionStoreReadError({ - threadId, - cause: "database unavailable", - }), - ), - ), - ), - // The failure is settled by the guarded write, not by another read. - shouldFinalizeRun: () => - Effect.fail( - new ProjectionStore.ProjectionStoreReadError({ - threadId, - cause: "database unavailable", - }), + close: Ref.update(closedSubscriptions, (n) => n + 1), + }), + events: Stream.never, + startTurn: () => Ref.update(providerStarts, (count) => count + 1), + } as unknown as ProviderAdapterV2SessionRuntime, + run: { id: runId, threadId, ordinal: 1, providerInstanceId } as OrchestrationV2Run, + rootNode: { + id: NodeId.make("node:run-execution-start-guard-read"), + } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make("checkpoint-scope:run-execution-start-guard-read"), + } as OrchestrationV2CheckpointScope, + providerThread: { + id: ProviderThreadId.make("provider-thread:run-execution-start-guard-read"), + driver, + } as OrchestrationV2ProviderThread, + attempt: { id: attemptId, providerTurnId: null } as OrchestrationV2RunAttempt, + attemptId, + providerTurnOrdinal: 1, + // The preparation check passes; the check right before the provider + // call cannot read the run. + shouldStartProviderTurn: () => + Ref.getAndUpdate(guardCalls, (calls) => calls + 1).pipe( + Effect.flatMap((calls) => + calls < failAt + ? Effect.succeed(true) + : Effect.fail( + new ProjectionStore.ProjectionStoreReadError({ + threadId, + cause: "database unavailable", + }), + ), ), - message: { - messageId: MessageId.make("message:run-execution-start-guard-read"), - text: "Start while the store is down.", - attachments: [], - createdBy: "user", - creationSource: "web", - }, - modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, - runtimePolicy: { - runtimeMode: "full-access", - interactionMode: "default", - cwd: process.cwd(), - approvalPolicy: "never", - sandboxPolicy: { - type: "readOnly", - access: { type: "fullAccess" }, - networkAccess: false, - }, + ), + // The failure is settled by the guarded write, not by another read. + shouldFinalizeRun: () => + Effect.fail( + new ProjectionStore.ProjectionStoreReadError({ + threadId, + cause: "database unavailable", + }), + ), + message: { + messageId: MessageId.make("message:run-execution-start-guard-read"), + text: "Start while the store is down.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, }, - }), - ); - assert.equal(Exit.isFailure(result), reattach); - }).pipe(Effect.provide(testLayer)); - - assert.equal(yield* Ref.get(guardCalls), failAt + 1); - assert.equal(yield* Ref.get(baselineCalls), checkpointFilesystem ? 1 : 0); - assert.equal(yield* Ref.get(closedSubscriptions), failAt); - if (reattach) assert.isEmpty(yield* Ref.get(writes)); - assert.equal(yield* Ref.get(providerStarts), 0); - const runUpdate = (yield* Ref.get(writes)).find((event) => event.type === "run.updated"); - assert.equal( - runUpdate?.type === "run.updated" ? runUpdate.payload.status : undefined, - reattach ? undefined : "failed", + }, + }), ); - }), + assert.equal(Exit.isFailure(result), reattach); + }).pipe(Effect.provide(testLayer)); + + assert.equal(yield* Ref.get(guardCalls), failAt + 1); + assert.equal(yield* Ref.get(baselineCalls), checkpointFilesystem ? 1 : 0); + assert.equal(yield* Ref.get(closedSubscriptions), failAt); + if (reattach) assert.isEmpty(yield* Ref.get(writes)); + assert.equal(yield* Ref.get(providerStarts), 0); + const runUpdate = (yield* Ref.get(writes)).find((event) => event.type === "run.updated"); + assert.equal( + runUpdate?.type === "run.updated" ? runUpdate.payload.status : undefined, + reattach ? undefined : "failed", + ); + }), ); it.effect( @@ -762,9 +766,7 @@ it.effect( appThread: { id: threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:compact-routing:${index}`), session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.never, startTurn: () => Effect.sync(() => { @@ -839,9 +841,7 @@ it.effect("refreshes MCP credential liveness before calling the provider", () => appThread: { id: threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make("session:run-execution-mcp-liveness"), session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.never, startTurn: () => Ref.update(order, (entries) => [...entries, "start-turn"]), } as unknown as ProviderAdapterV2SessionRuntime, @@ -953,9 +953,7 @@ it.effect("starts the provider when checkpoint baseline capture fails", () => appThread: { id: threadId } as OrchestrationV2AppThread, providerSessionId, session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.never, startTurn: () => Ref.update(providerStarts, (count) => count + 1), } as unknown as ProviderAdapterV2SessionRuntime, @@ -1092,9 +1090,7 @@ it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as co appThread: { id: threadId } as OrchestrationV2AppThread, providerSessionId, session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.never, startTurn: () => Ref.update(providerStarts, (count) => count + 1), } as unknown as ProviderAdapterV2SessionRuntime, @@ -1344,9 +1340,7 @@ it.effect("keeps ingesting owned child events after the root turn terminalizes", appThread: { id: threadId } as OrchestrationV2AppThread, providerSessionId, session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.fromIterable(events), startTurn: () => Effect.void, } as unknown as ProviderAdapterV2SessionRuntime, @@ -1727,9 +1721,7 @@ it.effect( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${key}`), session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.empty, // Session-wide stays true forever; the root must consult the // thread-scoped probe instead of being pinned by siblings. @@ -1947,9 +1939,7 @@ it.effect("drops late root provider-thread writes from a superseded attempt", () appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${key}`), session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.empty, hasPendingBackgroundWork: Effect.succeed(true), hasPendingBackgroundWorkForThread: () => Effect.succeed(true), @@ -2142,9 +2132,7 @@ it.effect( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${key}`), session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.empty, hasPendingBackgroundWork: Effect.succeed(true), hasPendingBackgroundWorkForThread: () => Effect.succeed(true), @@ -2305,9 +2293,7 @@ it.effect( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${key}`), session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.empty, // Session-wide stays true (sibling has work). Stop must use only // the scoped probe for this root's provider thread. @@ -2492,9 +2478,7 @@ it.effect( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make("session:subagent-interrupt-cascade"), session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.empty, subscribeEvents: Effect.succeed({ events: Stream.fromIterable([ @@ -2849,9 +2833,7 @@ it.effect( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make("session:subagent-link-survives-terminal"), session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.empty, subscribeEvents: Effect.succeed({ events: Stream.fromIterable([ @@ -3608,113 +3590,112 @@ function captureRootRunTermination(input: { yield* Effect.gen(function* () { const runExecution = yield* RunExecutionService.RunExecutionServiceV2; - yield* runExecution - .startRootRun({ - commandId: CommandId.make(`command:${input.key}`), - appThread: { id: ids.threadId } as OrchestrationV2AppThread, - providerSessionId: ProviderSessionId.make(`session:${input.key}`), - reattach: input.cloudReattach ?? false, - ...(input.nativeThreadHasTurns === undefined - ? {} - : { nativeThreadHasTurns: input.nativeThreadHasTurns }), - session: { - driver: input.cloudReattach ? ProviderDriverKind.make("kilo-cloud") : driver, - providerSession: { - capabilities: { - checkpointing: { - appCanCheckpointFilesystem: input.checkpointFilesystem ?? !input.cloudReattach, - }, + yield* runExecution.startRootRun({ + commandId: CommandId.make(`command:${input.key}`), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make(`session:${input.key}`), + reattach: input.cloudReattach ?? false, + ...(input.nativeThreadHasTurns === undefined + ? {} + : { nativeThreadHasTurns: input.nativeThreadHasTurns }), + session: { + driver: input.cloudReattach ? ProviderDriverKind.make("kilo-cloud") : driver, + providerSession: { + capabilities: { + checkpointing: { + appCanCheckpointFilesystem: input.checkpointFilesystem ?? !input.cloudReattach, }, }, - events: Stream.empty, - subscribeEvents: Effect.succeed({ - events: - input.events?.(ids) ?? - Stream.fromIterable([ - ...(input.seedOpenSubagent - ? [ - { type: "subagent.updated", driver, subagent: runningSubagent } as const, - { - type: "node.updated", - driver, - node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), - } as const, - { - type: "turn_item.updated", + }, + events: Stream.empty, + subscribeEvents: Effect.succeed({ + events: + input.events?.(ids) ?? + Stream.fromIterable([ + ...(input.seedOpenSubagent + ? [ + { type: "subagent.updated", driver, subagent: runningSubagent } as const, + { + type: "node.updated", + driver, + node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), + } as const, + { + type: "turn_item.updated", + driver, + turnItem: makeRunOwnedSubagentTurnItemFixture({ + ids, + providerInstanceId, + childThreadId: ids.childThreadId, driver, - turnItem: makeRunOwnedSubagentTurnItemFixture({ - ids, - providerInstanceId, - childThreadId: ids.childThreadId, - driver, - status: "running", - }), - } as const, - ] - : []), - rootTerminalEvent(ids, "interrupted"), - ] satisfies ReadonlyArray), - close: Deferred.succeed(ingestionDone, undefined), + status: "running", + }), + } as const, + ] + : []), + rootTerminalEvent(ids, "interrupted"), + ] satisfies ReadonlyArray), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: input.startTurn ?? (() => Effect.void), + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { + id: ids.rootNodeId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make(`checkpoint-scope:${input.key}`), + } as OrchestrationV2CheckpointScope, + providerThread: { + id: ids.providerThreadId, + driver, + } as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + providerTurnOrdinal: 1, + shouldFinalizeRun: input.shouldFinalizeRun, + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, }), - startTurn: input.startTurn ?? (() => Effect.void), - } as unknown as ProviderAdapterV2SessionRuntime, - run: { - id: ids.runId, - threadId: ids.threadId, - ordinal: 1, - providerInstanceId, - } as OrchestrationV2Run, - rootNode: { - id: ids.rootNodeId, - providerTurnId: ids.rootProviderTurnId, - } as OrchestrationV2ExecutionNode, - checkpointScope: { - id: CheckpointScopeId.make(`checkpoint-scope:${input.key}`), - } as OrchestrationV2CheckpointScope, - providerThread: { - id: ids.providerThreadId, - driver, - } as OrchestrationV2ProviderThread, - attempt: { - id: ids.attemptId, - providerTurnId: ids.rootProviderTurnId, - } as OrchestrationV2RunAttempt, - attemptId: ids.attemptId, - providerTurnOrdinal: 1, - shouldFinalizeRun: input.shouldFinalizeRun, - ...(input.hasUnpairedRunInterruptRequest === undefined - ? {} - : { - hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, - }), - message: { - messageId: MessageId.make(`message:${input.key}`), - text: "interrupt projection", - attachments: [], - createdBy: "user", - creationSource: "web", - }, - modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, - runtimePolicy: { - runtimeMode: "full-access", - interactionMode: "default", - cwd: process.cwd(), - approvalPolicy: "never", - sandboxPolicy: { - type: "readOnly", - access: { type: "fullAccess" }, - networkAccess: false, - }, + message: { + messageId: MessageId.make(`message:${input.key}`), + text: "interrupt projection", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, }, - }) - .pipe( - Effect.catch((error) => { - if (!input.cloudReattach) return Effect.fail(error); - startFailed = true; - return Effect.void; - }), - ); - }).pipe(Effect.provide(testLayer)); + }, + }); + }).pipe( + Effect.provide(testLayer), + Effect.catch((error) => { + if (!input.cloudReattach) return Effect.fail(error); + startFailed = true; + return Effect.void; + }), + ); yield* Deferred.await(ingestionDone); return { @@ -4107,9 +4088,7 @@ function runBackgroundItemScenario( appThread: { id: ids.threadId } as OrchestrationV2AppThread, providerSessionId: ProviderSessionId.make(`session:${key}`), session: { - providerSession: { - capabilities: { checkpointing: { appCanCheckpointFilesystem: true } }, - }, + providerSession: localWorkspaceSession, events: Stream.empty, subscribeEvents: Effect.gen(function* () { yield* options?.onSubscribe ?? Effect.void; diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index dddac128ef4b..f0e42b33a0d5 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -807,6 +807,8 @@ export const layer: Layer.Layer< return RunExecutionServiceV2.of({ startRootRun: (input) => Effect.gen(function* () { + // Without a local workspace the turn runs remotely and outlives this stream: + // losing it locally is not the turn's outcome, and recovery reattaches it. const checkpointFilesystem = input.session.providerSession.capabilities.checkpointing.appCanCheckpointFilesystem; // Startup failure and stream shutdown can report the same attempt. @@ -877,10 +879,7 @@ export const layer: Layer.Layer< }).pipe( Effect.catchCause((cause) => Effect.gen(function* () { - if ( - Cause.hasInterruptsOnly(cause) || - (input.session.driver === "kilo-cloud" && input.reattach) - ) { + if (Cause.hasInterruptsOnly(cause) || input.reattach === true) { return yield* Effect.failCause(cause); } yield* Effect.logError("orchestration V2 run preparation failed", { @@ -1143,12 +1142,9 @@ export const layer: Layer.Layer< return false; } const terminal = yield* Ref.get(terminalEvent); - // Non-completed terminals drop background tracking immediately. - if ( - terminal !== null && - terminal.status !== "completed" && - input.session.driver !== "kilo-cloud" - ) { + // Non-completed local terminals drop background tracking immediately; remote + // sandbox and billing state keep reporting after the turn ends. + if (terminal !== null && terminal.status !== "completed" && checkpointFilesystem) { return true; } const childProviderTurns = yield* Ref.get(activeChildProviderTurns); @@ -1304,7 +1300,7 @@ export const layer: Layer.Layer< cause, }).pipe( Effect.andThen( - finalized || input.session.driver === "kilo-cloud" + finalized || !checkpointFilesystem ? Effect.void : Ref.get(latestProviderThread).pipe( Effect.flatMap((providerThread) => @@ -1416,8 +1412,7 @@ export const layer: Layer.Layer< : input.session.startTurn(turnInput); yield* Effect.andThen(shouldStart, startTurn).pipe( Effect.catchCause((cause) => - input.session.driver === "kilo-cloud" && - (input.reattach || Cause.hasInterruptsOnly(cause)) + input.reattach === true || (!checkpointFilesystem && Cause.hasInterruptsOnly(cause)) ? stopProviderEvents.pipe( Effect.andThen( Effect.fail( From 062465c5e680c30681198a6baaf3619942fb31e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 16:46:01 +0000 Subject: [PATCH 33/44] test(kilo): compact the live adapter fixture Build streamed completion chunks and tool calls through two helpers instead of six inline copies, and open sessions through one helper. The live scenario and its assertions are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LguFPcf279M61rSEudqC4q --- .../Adapters/KiloAdapterV2.live.test.ts | 206 +++++------------- 1 file changed, 56 insertions(+), 150 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts index 51e4838bd15e..f41613794d32 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts @@ -41,6 +41,26 @@ import * as KiloAdapter from "./KiloAdapterV2.ts"; const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const binary = process.env.KILO_BIN; const layer = Layer.mergeAll(NodeServices.layer, IdAllocator.layer); +type RequestEvent = Extract; +const chunk = (choice: Record, extra?: Record) => + `data: ${JSON.stringify({ id: "chatcmpl-local", object: "chat.completion.chunk", created: 0, model: "test", choices: [{ index: 0, ...choice }], ...extra })}\n\n`; +/** A streamed completion that only calls one tool. */ +const toolCall = (name: string, args: unknown) => + chunk({ + delta: { + tool_calls: [ + { + index: 0, + id: `call_${name}`, + type: "function", + function: { name, arguments: JSON.stringify(args) }, + }, + ], + }, + finish_reason: null, + }) + + chunk({ delta: {}, finish_reason: "tool_calls" }) + + "data: [DONE]\n\n"; const inference = Effect.acquireRelease( Effect.promise(async () => { const requests: Array> = []; @@ -98,45 +118,12 @@ const inference = Effect.acquireRelease( messages.at(-1)?.role !== "tool" && !JSON.stringify(messages.at(-1) ?? null).includes("Child fixture reply") ) { - res.write( - `data: ${JSON.stringify({ - id: "chatcmpl-task", - object: "chat.completion.chunk", - created: 0, - model: "test", - choices: [ - { - index: 0, - delta: { - tool_calls: [ - { - index: 0, - id: "call_task", - type: "function", - function: { - name: "task", - arguments: JSON.stringify({ - description: "Local child", - prompt: "Child fixture reply", - subagent_type: "general", - }), - }, - }, - ], - }, - finish_reason: null, - }, - ], - })}\n\n`, - ); res.end( - `data: ${JSON.stringify({ - id: "chatcmpl-task", - object: "chat.completion.chunk", - created: 0, - model: "test", - choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], - })}\n\ndata: [DONE]\n\n`, + toolCall("task", { + description: "Local child", + prompt: "Child fixture reply", + subagent_type: "general", + }), ); return; } @@ -147,87 +134,41 @@ const inference = Effect.acquireRelease( JSON.stringify(messages.at(-1)).includes("Child fixture reply"))) && messages.at(-1)?.role !== "tool" ) { - const name = control.mode === "question" ? "question" : "bash"; - const args = - control.mode !== "question" - ? { - command: "printf kilo-approved > approval.txt", - description: "Write the local approval fixture", - } - : { + res.end( + control.mode === "question" + ? toolCall("question", { questions: [ { question: "Choose a color", header: "Color", multiple: true, - options: [ { label: "Blue", description: "Blue option" }, { label: "Red", description: "Red option" }, ], }, ], - }; - res.write( - `data: ${JSON.stringify({ - id: "chatcmpl-tool", - object: "chat.completion.chunk", - created: 0, - model: "test", - choices: [ - { - index: 0, - delta: { - tool_calls: [ - { - index: 0, - id: "call_fixture", - type: "function", - function: { name, arguments: JSON.stringify(args) }, - }, - ], - }, - finish_reason: null, - }, - ], - })}\n\n`, - ); - res.end( - `data: ${JSON.stringify({ - id: "chatcmpl-tool", - object: "chat.completion.chunk", - created: 0, - model: "test", - choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], - })}\n\ndata: [DONE]\n\n`, + }) + : toolCall("bash", { + command: "printf kilo-approved > approval.txt", + description: "Write the local approval fixture", + }), ); return; } if (control.mode === "text") res.write( - `data: ${JSON.stringify({ id: "chatcmpl-local", object: "chat.completion.chunk", created: 0, model: "test", choices: [{ index: 0, delta: { reasoning_content: "Fixture reasoning." }, finish_reason: null }] })}\n\n`, + chunk({ delta: { reasoning_content: "Fixture reasoning." }, finish_reason: null }), ); for (const text of control.mode === "json" ? [control.json] : ["Hello ", "from ", "local Kilo."]) - res.write( - `data: ${JSON.stringify({ - id: "chatcmpl-local", - object: "chat.completion.chunk", - created: 0, - model: "test", - choices: [{ index: 0, delta: { content: text }, finish_reason: null }], - })}\n\n`, - ); + res.write(chunk({ delta: { content: text }, finish_reason: null })); res.end( - `data: ${JSON.stringify({ - id: "chatcmpl-local", - object: "chat.completion.chunk", - created: 0, - model: "test", - choices: [{ index: 0, delta: {}, finish_reason: "stop" }], - usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, - })}\n\ndata: [DONE]\n\n`, + chunk( + { delta: {}, finish_reason: "stop" }, + { usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 } }, + ) + "data: [DONE]\n\n", ); }); }); @@ -355,12 +296,14 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> runtime: disconnectedRuntime, attachmentsDir: path.join(root, "attachments"), }); - const session = yield* adapter.openSession({ - threadId, - providerSessionId: ProviderSessionId.make("kilo-session"), - modelSelection, - runtimePolicy, - }); + const openAs = (name: string, kilo = adapter, thread = threadId) => + kilo.openSession({ + threadId: thread, + providerSessionId: ProviderSessionId.make(name), + modelSelection, + runtimePolicy, + }); + const session = yield* openAs("kilo-session"); const providerThread = yield* session.ensureThread({ threadId, modelSelection, @@ -369,10 +312,7 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> const seen: Adapter.ProviderAdapterV2Event[] = []; const questionShown = yield* Deferred.make(); let terminal = yield* Deferred.make(); - let interaction = - yield* Deferred.make< - Extract - >(); + let interaction = yield* Deferred.make(); yield* session.events.pipe( Stream.runForEach((event) => { seen.push(event); @@ -453,12 +393,7 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> ); assert.equal(restored.providerTurns.length, 1); assert.equal(restored.messages[0]!.id, firstInput.message.messageId); - const restoredSession = yield* adapter.openSession({ - threadId, - providerSessionId: ProviderSessionId.make("restored-session"), - modelSelection, - runtimePolicy, - }); + const restoredSession = yield* openAs("restored-session"); const restoredThread = yield* restoredSession.resumeThread({ providerThread: restored.providerThread, }); @@ -502,12 +437,7 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> providerTurnId: restored.providerTurns[0]!.id, targetThreadId: ThreadId.make("fork-thread"), }); - const forkSession = yield* adapter.openSession({ - threadId: ThreadId.make("fork-thread"), - providerSessionId: ProviderSessionId.make("fork-session"), - modelSelection, - runtimePolicy, - }); + const forkSession = yield* openAs("fork-session", adapter, ThreadId.make("fork-thread")); const forkThread = yield* forkSession.resumeThread({ providerThread: fork }); const forkHistory = yield* forkSession.readThreadSnapshot({ providerThread: forkThread }); assert.deepEqual( @@ -527,12 +457,7 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> assert.equal(rewound.messages.length, 2); assert.equal(rewound.messages[0]!.id, firstInput.message.messageId); assert.equal(rewound.messages[1]!.runId, firstInput.runId); - const rewindSession = yield* adapter.openSession({ - threadId, - providerSessionId: ProviderSessionId.make("rewind-resume"), - modelSelection, - runtimePolicy, - }); + const rewindSession = yield* openAs("rewind-resume"); const rewindThread = yield* rewindSession.resumeThread({ providerThread: rewound.providerThread, }); @@ -559,10 +484,7 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> assert.equal(empty.messages.length, 0); for (const decision of ["decline", "accept"] as const) { terminal = yield* Deferred.make(); - interaction = - yield* Deferred.make< - Extract - >(); + interaction = yield* Deferred.make(); model.control.mode = "approval"; yield* session.startTurn({ ...firstInput, @@ -580,10 +502,7 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> assert.equal(yield* fs.exists(path.join(root, "approval.txt")), decision === "accept"); } terminal = yield* Deferred.make(); - interaction = - yield* Deferred.make< - Extract - >(); + interaction = yield* Deferred.make(); dropInteraction = true; model.control.mode = "question"; yield* session.startTurn({ @@ -830,10 +749,7 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> dropInteraction = action === "accept"; model.control.mode = "subagent-approval"; terminal = yield* Deferred.make(); - interaction = - yield* Deferred.make< - Extract - >(); + interaction = yield* Deferred.make(); const startIndex = seen.length; yield* session.startTurn({ ...firstInput, @@ -905,12 +821,7 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> ); if (saved?.type !== "provider_thread.updated") throw new Error("Missing durable interruption metadata"); - const fresh = yield* adapter.openSession({ - threadId, - providerSessionId: ProviderSessionId.make("after-stop"), - modelSelection, - runtimePolicy, - }); + const fresh = yield* openAs("after-stop"); const resumed = yield* fresh.resumeThread({ providerThread: saved.providerThread }); const history = yield* fresh.readThreadSnapshot({ providerThread: resumed }); assert.equal( @@ -952,12 +863,7 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> cwd: root, runtime, }); - const otherSession = yield* otherAccount.openSession({ - threadId, - providerSessionId: ProviderSessionId.make("other"), - modelSelection, - runtimePolicy, - }); + const otherSession = yield* openAs("other", otherAccount); yield* otherSession.resumeThread({ providerThread }).pipe(Effect.flip); yield* otherSession .ensureThread({ From bfd9d45d46f2b3045ce2b34d698cfc841dfe8b8f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 16:50:17 +0000 Subject: [PATCH 34/44] refactor(orchestration): keep upstream's start-failure chain intact Branch on a remote start failure from the start's Exit before the existing catch, and gate the baseline and pull-request refresh with Effect.when, so upstream's blocks keep their shape. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LguFPcf279M61rSEudqC4q --- .../orchestration-v2/RunExecutionService.ts | 154 +++++++++--------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index f0e42b33a0d5..c3dc28d272f6 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -813,10 +813,8 @@ export const layer: Layer.Layer< input.session.providerSession.capabilities.checkpointing.appCanCheckpointFilesystem; // Startup failure and stream shutdown can report the same attempt. const refreshAfterTurn = yield* Effect.cached( - (checkpointFilesystem - ? finalizationObserver.refreshAfterTurn(input.appThread.projectId) - : Effect.void - ).pipe( + finalizationObserver.refreshAfterTurn(input.appThread.projectId).pipe( + Effect.when(Effect.succeed(checkpointFilesystem)), Effect.catchCause((cause) => Effect.logWarning("failed to refresh pull requests after run termination", { threadId: input.run.threadId, @@ -853,22 +851,22 @@ export const layer: Layer.Layer< .responseStreamingMode, ), ); - if (checkpointFilesystem) - yield* checkpointService - .captureBaseline({ - scope: input.checkpointScope, - ordinalWithinScope: Math.max(0, input.run.ordinal - 1), - }) - .pipe( - Effect.catchCause((cause) => - Cause.hasInterruptsOnly(cause) - ? Effect.failCause(cause) - : Effect.logWarning( - "orchestration V2 checkpoint baseline capture failed; starting provider without a baseline", - { runId: input.run.id }, - ), - ), - ); + yield* checkpointService + .captureBaseline({ + scope: input.checkpointScope, + ordinalWithinScope: Math.max(0, input.run.ordinal - 1), + }) + .pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.failCause(cause) + : Effect.logWarning( + "orchestration V2 checkpoint baseline capture failed; starting provider without a baseline", + { runId: input.run.id }, + ), + ), + Effect.when(Effect.succeed(checkpointFilesystem)), + ); if ( input.shouldStartProviderTurn !== undefined && !(yield* input.shouldStartProviderTurn()) @@ -1410,70 +1408,72 @@ export const layer: Layer.Layer< }), )) : input.session.startTurn(turnInput); - yield* Effect.andThen(shouldStart, startTurn).pipe( + const started = yield* Effect.exit(Effect.andThen(shouldStart, startTurn)); + // A remote turn outlives a lost start; recovery reattaches it instead of failing the run. + if ( + Exit.isFailure(started) && + (input.reattach === true || + (!checkpointFilesystem && Cause.hasInterruptsOnly(started.cause))) + ) { + yield* stopProviderEvents; + return yield* new RunExecutionStartError({ + commandId: input.commandId, + runId: input.run.id, + cause: started.cause, + }); + } + yield* started.pipe( Effect.catchCause((cause) => - input.reattach === true || (!checkpointFilesystem && Cause.hasInterruptsOnly(cause)) - ? stopProviderEvents.pipe( - Effect.andThen( - Effect.fail( - new RunExecutionStartError({ - commandId: input.commandId, - runId: input.run.id, - cause, - }), - ), - ), - ) - : Effect.logError("orchestration V2 provider turn start failed", { - runId: input.run.id, - cause, - }).pipe( - Effect.andThen(stopProviderEvents), - Effect.andThen(Ref.get(latestProviderThread)), - Effect.flatMap((providerThread) => - Ref.get(latestTurnItemOrdinal).pipe( - Effect.flatMap((latestItemOrdinal) => - Ref.get(openRunOwnedSubagents).pipe( - Effect.flatMap((openSubagents) => - writeFinalRunEvents({ - checkpointFilesystem, - run: input.run, - rootNode: input.rootNode, - checkpointScope: input.checkpointScope, - providerThread, - attempt: input.attempt, - // Ownership reads can fail; fence the failure in the write transaction. - writeIfRunCurrent: { - activeAttemptId: input.attempt.id, - expectedStatus: "running", - }, - openRunOwnedSubagents: openSubagents, - terminal: makeFailedTerminalEvent( - makeProviderFailure({ - cause: Cause.squash(cause), - class: Exit.isFailure(shouldStart) - ? "unknown" - : "provider_error", - }), - latestItemOrdinal + 1, - ), - failureItemPersisted: false, - refreshAfterTurn, + Effect.logError("orchestration V2 provider turn start failed", { + runId: input.run.id, + cause, + }).pipe( + Effect.andThen(stopProviderEvents), + Effect.andThen(Ref.get(latestProviderThread)), + Effect.flatMap((providerThread) => + Ref.get(latestTurnItemOrdinal).pipe( + Effect.flatMap((latestItemOrdinal) => + Ref.get(openRunOwnedSubagents).pipe( + Effect.flatMap((openSubagents) => + writeFinalRunEvents({ + checkpointFilesystem, + run: input.run, + rootNode: input.rootNode, + checkpointScope: input.checkpointScope, + providerThread, + attempt: input.attempt, + // Checked in the write transaction, not by another + // read that can fail like the one before the start. + writeIfRunCurrent: { + activeAttemptId: input.attempt.id, + expectedStatus: "running", + }, + openRunOwnedSubagents: openSubagents, + terminal: makeFailedTerminalEvent( + makeProviderFailure({ + cause: Cause.squash(cause), + // A failed ownership read is not the provider's fault. + class: Exit.isFailure(shouldStart) ? "unknown" : "provider_error", }), + latestItemOrdinal + 1, ), - ), + failureItemPersisted: false, + refreshAfterTurn, + }), ), ), ), - Effect.mapError( - (writeCause) => - new RunExecutionStartError({ - commandId: input.commandId, - runId: input.run.id, - cause: { start: cause, write: writeCause }, - }), - ), ), + ), + Effect.mapError( + (writeCause) => + new RunExecutionStartError({ + commandId: input.commandId, + runId: input.run.id, + cause: { start: cause, write: writeCause }, + }), + ), + ), ), ); }), From 190ebf2eef252898e419740e629ecf0642672cb0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 16:51:04 +0000 Subject: [PATCH 35/44] test(kilo): check foreign permission and question replies in one case Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LguFPcf279M61rSEudqC4q --- .../provider/kilo/KiloSessionClient.test.ts | 61 +++++++------------ 1 file changed, 21 insertions(+), 40 deletions(-) diff --git a/apps/server/src/provider/kilo/KiloSessionClient.test.ts b/apps/server/src/provider/kilo/KiloSessionClient.test.ts index 46d08ff9c8b9..e260b3674a5b 100644 --- a/apps/server/src/provider/kilo/KiloSessionClient.test.ts +++ b/apps/server/src/provider/kilo/KiloSessionClient.test.ts @@ -227,46 +227,27 @@ describe("Kilo native SDK boundary", () => { expect(admissions).toBe(1); }); - it("does not answer another session's approval even when its request id is known", async () => { - let replies = 0; - const client = await withClient((req, res) => { - if (req.method === "POST") replies++; - json( - res, - req.url!.startsWith("/permission") - ? [ - { - id: "request-one", - sessionID: "ses_other", - permission: "bash", - patterns: ["*"], - metadata: {}, - always: [], - }, - ] - : { id: ref.sessionId, directory }, - ); - }); - const error = await run(client.replyPermission(ref, "request-one", "once").pipe(Effect.flip)); - expect(error.reason).toBe("wrong_owner"); - expect(replies).toBe(0); - }); - - it("does not answer another session's question", async () => { - let replies = 0; - const client = await withClient((req, res) => { - if (req.method === "POST") replies++; - json( - res, - req.url!.startsWith("/question") - ? [{ id: "question-one", sessionID: "ses_other", questions: [] }] - : { id: ref.sessionId, directory }, - ); - }); - const error = await run(client.replyQuestion(ref, "question-one", [["yes"]]).pipe(Effect.flip)); - expect(error.reason).toBe("wrong_owner"); - expect(replies).toBe(0); - }); + it.each(["permission", "question"] as const)( + "does not answer another session's %s even when its request id is known", + async (kind) => { + let replies = 0; + const client = await withClient((req, res) => { + if (req.method === "POST") replies++; + json( + res, + req.url!.startsWith(`/${kind}`) + ? [{ id: "request", sessionID: "ses_other" }] + : { id: ref.sessionId, directory }, + ); + }); + const reply = + kind === "permission" + ? client.replyPermission(ref, "request", "once") + : client.replyQuestion(ref, "request", [["yes"]]); + expect((await run(reply.pipe(Effect.flip))).reason).toBe("wrong_owner"); + expect(replies).toBe(0); + }, + ); it("filters interleaved events and reports EOF without reconnecting or claiming completion", async () => { let subscriptions = 0; From fa50b96ba194abbbd8626e95fff9266e24fb26e3 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Mon, 5 Oct 2026 19:22:28 +0000 Subject: [PATCH 36/44] fix(kilo): recover cloud runs without resubmitting tasks Hold queued cloud runs after restart, restore reattach eligibility, and settle proven missing intents. Resume failed observers through a guarded durable effect and keep unavailable cloud drafts free of local controls. Assisted by GPT-6 in the Codex cloud harness on behalf of @thomasbrugman. --- .../Adapters/KiloCloudAdapterV2.test.ts | 19 +++++ .../Adapters/KiloCloudAdapterV2.ts | 11 ++- apps/server/src/orchestration-v2/EventSink.ts | 9 +- .../FoundationPersistence.test.ts | 56 ++++++++----- .../src/orchestration-v2/ProviderAdapter.ts | 2 + .../ProviderRuntimeRecoveryService.test.ts | 37 ++++++-- .../ProviderRuntimeRecoveryService.ts | 34 ++++---- .../RunExecutionService.test.ts | 84 +++++++++++++++++-- .../orchestration-v2/RunExecutionService.ts | 72 ++++++++++++++++ apps/web/src/components/ChatView.tsx | 2 +- 10 files changed, 271 insertions(+), 55 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts index 3a81d0db12f6..1679c1148cfa 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.test.ts @@ -1357,6 +1357,25 @@ const uncertainAdmission = Effect.gen(function* () { return { remote, directory }; }); +it.live("proves a missing recovery intent was not submitted without creating a task", () => + Effect.gen(function* () { + const remote = yield* fixture; + const harness = yield* admissionHarness(remote, yield* authorizedDirectory); + const { runtime, thread } = yield* harness.open; + const result = yield* Effect.result( + runtime.startTurn({ ...harness.turn(thread), reattach: true }), + ); + assert.equal(result._tag, "Failure"); + if (result._tag !== "Failure") return; + assert.equal(result.failure._tag, "ProviderAdapterTurnStartError"); + if (result.failure._tag !== "ProviderAdapterTurnStartError") return; + assert.isTrue(result.failure.notSubmitted); + assert.include(String(result.failure.cause), "No durable cloud intent"); + assert.equal(remote.submissions(), 0); + assert.isEmpty(yield* harness.journal.read); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, IdAllocator.layer))), +); + it.live.each([ "404", "503", diff --git a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts index a84638100b79..7c207c117037 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloCloudAdapterV2.ts @@ -1112,9 +1112,14 @@ export const make = Effect.fn("KiloCloudAdapterV2.make")(function* (options: { entry.providerTurn.runAttemptId === request.attemptId, ); if (!saved) - return yield* error( - "No durable cloud intent exists for this run. No task was resubmitted.", - ); + return yield* new Adapter.ProviderAdapterTurnStartError({ + driver, + threadId: request.threadId, + providerThreadId: request.providerThread.id, + runId: request.runId, + notSubmitted: true, + cause: "No durable cloud intent exists for this run. No task was submitted.", + }); active = saved; yield* emit({ type: "provider_turn.updated", diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 71b6f0ea994f..ca20e8f7217d 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -90,6 +90,7 @@ export interface EventSinkV2Shape { readonly activeAttemptId: RunAttemptId; readonly expectedStatus: OrchestrationV2Run["status"]; readonly events: ReadonlyArray; + readonly effects?: ReadonlyArray; }) => Effect.Effect< { readonly committed: boolean; @@ -437,9 +438,15 @@ const baseLayer: Layer.Layer< events: normalized, }); yield* applyStoredEvents(storedEvents); + yield* effectOutbox.enqueue(input.effects ?? []); return { committed: true as const, storedEvents }; }), - (result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void), + (result) => + Effect.gen(function* () { + if (!result.committed) return; + if (input.effects?.length) yield* effectOutbox.notifyAvailable(input.effects.length); + yield* publishStoredEvents(result.storedEvents); + }), ); }, ); diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index dc7eacc59ba5..d59e74681ebb 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -1509,9 +1509,11 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { }).pipe(Effect.provide(Layer.fresh(TestLayer))), ); - it.effect("does not publish a stale provider start after an interrupt wins", () => + it.effect.each([true, false])("guards recovery effects (%s)", (interrupted) => Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const commandId = CommandId.make("command:foundation-guarded-recovery"); const projectionStore = yield* ProjectionStore.ProjectionStoreV2; const now = yield* DateTime.now; const threadId = ThreadId.make("thread:foundation-stale-provider-start"); @@ -1566,6 +1568,14 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { runId, activeAttemptId: attemptId, expectedStatus: "starting", + effects: [ + { + id: "effect:foundation-guarded-recovery", + commandId, + threadId, + request: { type: "provider-turn.reattach", runId }, + }, + ], events: [ { id: EventId.make("event:foundation-stale-provider-start:running"), @@ -1586,32 +1596,34 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { yield* Deferred.await(reachedPrecommitGap); const interruptedAt = yield* DateTime.now; - yield* eventSink.write({ - events: [ - { - id: EventId.make("event:foundation-stale-provider-start:cancelled"), - type: "run.updated", - threadId, - runId, - providerInstanceId, - occurredAt: interruptedAt, - payload: { - ...startingRun, - status: "cancelled", - completedAt: interruptedAt, + if (interrupted) + yield* eventSink.write({ + events: [ + { + id: EventId.make("event:foundation-stale-provider-start:cancelled"), + type: "run.updated", + threadId, + runId, + providerInstanceId, + occurredAt: interruptedAt, + payload: { + ...startingRun, + status: "cancelled", + completedAt: interruptedAt, + }, }, - }, - ], - }); + ], + }); yield* Deferred.succeed(releaseStaleStart, undefined); const staleResult = yield* Fiber.join(staleStartFiber); - assert.isFalse(staleResult.committed); - assert.deepEqual(staleResult.storedEvents, []); - assert.equal(yield* Ref.get(providerStartCount), 0); + assert.equal(staleResult.committed, !interrupted); + assert.equal(staleResult.storedEvents.length, interrupted ? 0 : 1); + assert.equal(yield* Ref.get(providerStartCount), interrupted ? 0 : 1); + assert.equal((yield* outbox.listByCommandId(commandId)).length, interrupted ? 0 : 1); const projection = yield* projectionStore.getThreadProjection(threadId); - assert.equal(projection.runs[0]?.status, "cancelled"); - }), + assert.equal(projection.runs[0]?.status, interrupted ? "cancelled" : "running"); + }).pipe(Effect.provide(Layer.fresh(TestLayer))), ); it.effect("guards post-terminal provider-thread writes by attempt and run ordinal", () => diff --git a/apps/server/src/orchestration-v2/ProviderAdapter.ts b/apps/server/src/orchestration-v2/ProviderAdapter.ts index a42cb8200cee..aa6e9143a538 100644 --- a/apps/server/src/orchestration-v2/ProviderAdapter.ts +++ b/apps/server/src/orchestration-v2/ProviderAdapter.ts @@ -266,6 +266,8 @@ export class ProviderAdapterTurnStartError extends Schema.TaggedError { +it.effect.each(["pending", "running"] as const)( + "preserves remote work and reuses a %s reattach while still cancelling local work", + (effectStatus) => { const threadId = ThreadId.make("mixed-cloud-local"); const cloudThread = ProviderThreadId.make("remote-thread"); const localThread = ProviderThreadId.make("local-thread"); const cloudTurn = ProviderTurnId.make("remote-turn"); const localTurn = ProviderTurnId.make("local-turn"); const cloudRun = RunId.make("remote-run"); + const queuedRun = RunId.make("remote-queued-run"); + const heldRun = RunId.make("remote-held-run"); const localRun = RunId.make("local-run"); const cloudInstance = ProviderInstanceId.make("cloud-instance"); const localInstance = ProviderInstanceId.make("local-instance"); @@ -1301,6 +1303,15 @@ it.effect( ], providerSessions: [], runs: [ + ...[queuedRun, heldRun].map((id) => ({ + id, + providerThreadId: cloudThread, + providerInstanceId: cloudInstance, + status: "queued", + queueHeld: id === heldRun, + rootNodeId: null, + activeAttemptId: null, + })), { id: cloudRun, providerThreadId: cloudThread, @@ -1376,7 +1387,7 @@ it.effect( () => pending .filter((effect) => effect.commandId === id) - .map((effect) => ({ ...effect, status: "pending" })) as never, + .map((effect) => ({ ...effect, status: effectStatus })) as never, ), reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }), cancelUnsettled: () => Effect.succeed([]), @@ -1390,14 +1401,30 @@ it.effect( yield* service.reconcile("shutdown"); assert.isTrue(events.some((run) => run.id === localRun && run.status === "cancelled")); assert.isFalse(events.some((run) => run.id === cloudRun)); + assert.isTrue(events.some((run) => run.id === queuedRun && run.queueHeld === true)); + assert.isFalse(events.some((run) => run.id === heldRun)); assert.equal(requests.length, 0); + events.length = 0; yield* service.reconcile("startup"); yield* service.reconcile("startup"); assert.equal( pending.filter((effect) => effect.request.type === "provider-turn.reattach").length, 1, ); - assert.isTrue(events.some((run) => run.id === cloudRun && run.status === "starting")); + assert.equal( + events.filter((run) => run.id === cloudRun && run.status === "starting").length, + 2, + ); + assert.isTrue(events.some((run) => run.id === queuedRun && run.queueHeld === true)); + assert.isFalse(events.some((run) => run.id === heldRun)); + events.length = 0; + Object.assign( + projection.runs.find((run) => run.id === cloudRun)!, + { status: "starting" }, + ); + yield* service.reconcile("startup"); + assert.isFalse(events.some((run) => run.id === cloudRun)); + assert.equal(pending.length, 1); assert.isFalse(events.some((run) => run.id === cloudRun && run.status === "cancelled")); assert.equal(requests.length, 0); }).pipe(Effect.provide(layer)); diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 97c9c0ea0960..1d110b028158 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -190,9 +190,10 @@ export const make = Effect.gen(function* () { .filter((thread) => thread.driver === "kilo-cloud") .map((thread) => thread.id), ); - const cloudRuns = nonterminalRuns(projection).filter( - (run) => run.providerThreadId !== null && cloudThreads.has(run.providerThreadId), - ); + const cloudRuns = [ + ...nonterminalRuns(projection), + ...projection.runs.filter((run) => run.status === "queued"), + ].filter((run) => run.providerThreadId !== null && cloudThreads.has(run.providerThreadId)); if (cloudThreads.size > 0) { const events: OrchestrationV2DomainEvent[] = []; const effects: EffectOutbox.PendingOrchestrationEffectV2[] = []; @@ -210,20 +211,19 @@ export const make = Effect.gen(function* () { (cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }), ), ); - if ( - existing.some( - (effect) => - effect.request.type === "provider-turn.reattach" && - (effect.status === "pending" || effect.status === "running"), - ) - ) - continue; - effects.push({ - id: `effect:cloud-reattach:${run.id}:${run.activeAttemptId}:${DateTime.formatIso(now)}`, - commandId, - threadId: projection.thread.id, - request: { type: "provider-turn.reattach", runId: run.id }, - }); + const reuse = existing.some( + (effect) => + effect.request.type === "provider-turn.reattach" && + (effect.status === "pending" || effect.status === "running"), + ); + if (reuse && run.status === "starting") continue; + if (!reuse) + effects.push({ + id: `effect:cloud-reattach:${run.id}:${run.activeAttemptId}:${DateTime.formatIso(now)}`, + commandId, + threadId: projection.thread.id, + request: { type: "provider-turn.reattach", runId: run.id }, + }); } events.push({ id: yield* ids.allocate diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 89f0b520e0e5..e27ee3174100 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -730,12 +730,11 @@ it.effect.each(readFailures)("ownership read failure $driverName/$reattach/$fail assert.equal(yield* Ref.get(guardCalls), failAt + 1); assert.equal(yield* Ref.get(baselineCalls), checkpointFilesystem ? 1 : 0); assert.equal(yield* Ref.get(closedSubscriptions), failAt); - if (reattach) assert.isEmpty(yield* Ref.get(writes)); assert.equal(yield* Ref.get(providerStarts), 0); const runUpdate = (yield* Ref.get(writes)).find((event) => event.type === "run.updated"); assert.equal( runUpdate?.type === "run.updated" ? runUpdate.payload.status : undefined, - reattach ? undefined : "failed", + reattach ? "starting" : "failed", ); }), ); @@ -3436,11 +3435,78 @@ it.effect( assert.deepEqual(delivered, [{ reattach: true, nativeThreadHasTurns }]); assert.isTrue(result.startFailed); assert.deepEqual(result.written, []); - assert.deepEqual(result.observed, []); + assert.deepEqual(result.observed, ["run:starting"]); + assert.isEmpty(result.effects); } }), ); +it.effect("settles a cloud reattach as failed when the adapter proves it was never submitted", () => + Effect.gen(function* () { + const result = yield* captureRootRunTermination({ + key: "cloud-reattach-missing-intent", + cloudReattach: true, + shouldFinalizeRun: () => Effect.succeed(true), + events: () => Stream.never, + startTurn: (input) => + Effect.fail( + new ProviderAdapterTurnStartError({ + driver: ProviderDriverKind.make("kilo-cloud"), + threadId: input.threadId, + providerThreadId: input.providerThread.id, + runId: input.runId, + notSubmitted: true, + cause: "No durable cloud intent exists for this run. No task was submitted.", + }), + ), + }); + assert.isFalse(result.startFailed); + assert.deepEqual(result.observed, ["run:failed"]); + assert.include( + result.written.find((item) => item.type === "error")?.failure.message ?? "", + "No task was submitted", + ); + assert.isEmpty(result.effects); + assert.equal(result.baselineCalls, 0); + }), +); + +it.effect.each([true, false])( + "reattaches a failed cloud observer only while its run is current: %s", + (runCurrent) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const result = yield* captureRootRunTermination({ + key: `cloud-ingestion-failure-${runCurrent}`, + cloudReattach: true, + runCurrent, + shouldFinalizeRun: () => Effect.succeed(true), + startTurn: () => Deferred.succeed(started, undefined), + events: () => + Stream.unwrap( + Deferred.await(started).pipe( + Effect.as( + Stream.fail( + new ProviderAdapterEventStreamError({ + driver: ProviderDriverKind.make("kilo-cloud"), + providerSessionId: ProviderSessionId.make("cloud-session-lost"), + cause: "provider observer lost", + }), + ), + ), + ), + ), + }); + assert.deepEqual(result.observed, runCurrent ? ["run:starting"] : []); + assert.deepEqual( + result.effects.map((effect) => effect.request.type), + runCurrent ? ["provider-turn.reattach"] : [], + ); + assert.isEmpty(result.written); + assert.equal(result.baselineCalls, 0); + }), +); + it.effect("refreshes pull requests only once when startup failure closes its event stream", () => Effect.gen(function* () { const ingestionStarted = yield* Deferred.make(); @@ -3505,6 +3571,7 @@ it.effect.each([true, false])( function captureRootRunTermination(input: { readonly key: string; + readonly runCurrent?: boolean; readonly cloudReattach?: boolean; readonly nativeThreadHasTurns?: boolean; readonly checkpointFilesystem?: boolean; @@ -3568,9 +3635,14 @@ function captureRootRunTermination(input: { Effect.as([]), ), writeIfRunCurrent: (payload) => - captureFinalEvents(payload.events).pipe( - Effect.as({ committed: true, storedEvents: [] }), - ), + input.runCurrent === false + ? Effect.succeed({ committed: false, storedEvents: [] }) + : captureFinalEvents(payload.events).pipe( + Effect.andThen( + Ref.update(effects, (current) => [...current, ...(payload.effects ?? [])]), + ), + Effect.as({ committed: true, storedEvents: [] }), + ), }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index c3dc28d272f6..f28ec458c5bf 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -75,6 +75,8 @@ export interface InheritedBackgroundTurnItemRoute { readonly runId: OrchestrationV2Run["id"]; } +const isTurnStartError = Schema.is(ProviderAdapterTurnStartError); + type ProviderTerminalEvent = Extract; function isTerminalProviderTurnStatus(status: OrchestrationV2ProviderTurn["status"]): boolean { @@ -811,6 +813,56 @@ export const layer: Layer.Layer< // losing it locally is not the turn's outcome, and recovery reattaches it. const checkpointFilesystem = input.session.providerSession.capabilities.checkpointing.appCanCheckpointFilesystem; + const prepareCloudReattach = Effect.fn("RunExecutionService.prepareCloudReattach")( + function* (enqueue: boolean) { + const now = yield* DateTime.now; + const commandId = CommandId.make( + `command:cloud-reattach:${input.run.id}:${input.attempt.id}`, + ); + yield* eventSink.writeIfRunCurrent({ + threadId: input.run.threadId, + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedStatus: "running", + events: [ + { + id: yield* idAllocator.allocate.event({ + threadId: input.run.threadId, + commandId, + }), + type: "run.updated", + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: now, + payload: { ...input.run, status: "starting" }, + }, + ], + effects: enqueue + ? [ + { + id: `effect:cloud-reattach:${input.run.id}:${input.attempt.id}:${DateTime.formatIso(now)}`, + commandId, + threadId: input.run.threadId, + request: { + type: "provider-turn.reattach", + runId: input.run.id, + }, + }, + ] + : [], + }); + }, + Effect.mapError( + (cause) => + new RunExecutionStartError({ + commandId: input.commandId, + runId: input.run.id, + cause, + }), + ), + ); // Startup failure and stream shutdown can report the same attempt. const refreshAfterTurn = yield* Effect.cached( finalizationObserver.refreshAfterTurn(input.appThread.projectId).pipe( @@ -878,6 +930,8 @@ export const layer: Layer.Layer< Effect.catchCause((cause) => Effect.gen(function* () { if (Cause.hasInterruptsOnly(cause) || input.reattach === true) { + if (input.reattach === true && !Cause.hasInterruptsOnly(cause)) + yield* prepareCloudReattach(false); return yield* Effect.failCause(cause); } yield* Effect.logError("orchestration V2 run preparation failed", { @@ -1297,6 +1351,13 @@ export const layer: Layer.Layer< runId: input.run.id, cause, }).pipe( + Effect.andThen( + !finalized && + input.session.driver === "kilo-cloud" && + !Cause.hasInterruptsOnly(cause) + ? prepareCloudReattach(true) + : Effect.void, + ), Effect.andThen( finalized || !checkpointFilesystem ? Effect.void @@ -1409,13 +1470,18 @@ export const layer: Layer.Layer< )) : input.session.startTurn(turnInput); const started = yield* Effect.exit(Effect.andThen(shouldStart, startTurn)); + const startError = Exit.isFailure(started) ? Cause.squash(started.cause) : undefined; + const notSubmitted = isTurnStartError(startError) && startError.notSubmitted === true; // A remote turn outlives a lost start; recovery reattaches it instead of failing the run. if ( Exit.isFailure(started) && + !notSubmitted && (input.reattach === true || (!checkpointFilesystem && Cause.hasInterruptsOnly(started.cause))) ) { yield* stopProviderEvents; + if (input.reattach === true && !Cause.hasInterruptsOnly(started.cause)) + yield* prepareCloudReattach(false); return yield* new RunExecutionStartError({ commandId: input.commandId, runId: input.run.id, @@ -1452,6 +1518,12 @@ export const layer: Layer.Layer< terminal: makeFailedTerminalEvent( makeProviderFailure({ cause: Cause.squash(cause), + ...(notSubmitted + ? { + message: + "No task was submitted. Send a new message to try again.", + } + : {}), // A failed ownership read is not the provider's fault. class: Exit.isFailure(shouldStart) ? "unknown" : "provider_error", }), diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 364185aa3ee8..1f78ee162a23 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -3196,7 +3196,7 @@ export default function ChatView(props: ChatViewProps) { const selectedProvider = selectedProviderEntry?.driverKind ?? requestedDriverKind; const activeProviderInstanceId = selectedProviderEntry?.instanceId ?? null; const activeProviderStatus = selectedProviderEntry?.snapshot ?? null; - const isCloudComposer = activeProviderStatus?.driver === "kilo-cloud"; + const isCloudComposer = selectedProvider === "kilo-cloud"; const persistedProviderThread = serverProjection?.providerThreads.find( (thread) => thread.id === serverProjection.thread.activeProviderThreadId, ); From 0c54fdb97297533693ffe6f9099c703df1ac70ac Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Mon, 5 Oct 2026 19:40:24 +0000 Subject: [PATCH 37/44] fix(web): keep cloud sends clear of local workspace actions Assisted by GPT-6 in the Codex cloud harness on behalf of @thomasbrugman. --- apps/web/src/components/ChatView.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 1f78ee162a23..07627fc7aa73 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -6715,7 +6715,7 @@ export default function ChatView(props: ChatViewProps) { activeProjectSettings.settings.newWorktreesStartFromOrigin) : false; const sendEnvMode = resolveSendEnvMode({ - requestedEnvMode: envMode, + requestedEnvMode: isCloudComposer ? "local" : envMode, isGitRepo, }); const localCheckoutBranchMismatch = useMemo( @@ -7899,7 +7899,7 @@ export default function ChatView(props: ChatViewProps) { } const scriptId = projectScriptIdFromCommand(command); - if (!scriptId || !activeProject) return; + if (isCloudThread || !scriptId || !activeProject) return; const script = activeProjectScripts.find((entry) => entry.id === scriptId); if (!script) return; event.preventDefault(); From 51008c21a9e7e31e3be6df9f870be14165f86a2a Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Mon, 5 Oct 2026 19:58:27 +0000 Subject: [PATCH 38/44] fix(web): isolate cloud targets from local workspace UI Assisted by GPT-6 in the Codex cloud harness on behalf of @thomasbrugman. --- apps/web/src/components/ChatView.tsx | 41 ++++++++++++++++------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 07627fc7aa73..33b9038af736 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -2279,7 +2279,6 @@ export default function ChatView(props: ChatViewProps) { [activeKnownTerminalIds, panelTerminalIds], ); const previewPanelOpen = activeRightPanelKind === "preview" && isPreviewSupportedInRuntime(); - const rightPanelOpen = rightPanelState.isOpen; const { active: panelAnimationsActive, durationMs: panelAnimationDurationMs } = usePanelAnimationSettings(); const activeTerminalDrawerPresence = usePanelPresence( @@ -2297,26 +2296,18 @@ export default function ChatView(props: ChatViewProps) { [activeRightPanelSurface, rightPanelState.surfaces], ); const rightPanelPresence = usePanelPresence( - rightPanelOpen && activeThreadRef !== null, + rightPanelState.isOpen && activeThreadRef !== null, rightPanelPresenceValue, panelAnimationsActive, activeThreadKey, panelAnimationDurationMs, ); - const rightPanelPresent = rightPanelPresence.present; - const rightPanelControlsInPanel = - shouldUsePlanSidebarSheet && rightPanelPresent && rightPanelOpen; - const rightPanelControlsAtRoot = rightPanelPresent && !shouldUsePlanSidebarSheet; const renderedRightPanelSurface = rightPanelPresence.value?.activeSurface ?? null; const renderedRightPanelSurfaces = rightPanelPresence.value?.surfaces ?? []; const previewMiniPlayerVisible = shouldRenderPreviewMiniPlayer( activePreviewMiniPlayer?.source ?? null, renderedRightPanelSurface, ); - const canMaximizeRightPanel = rightPanelOpen && !shouldUsePlanSidebarSheet; - const rightPanelMaximized = - canMaximizeRightPanel && maximizedRightPanelThreadKey === routeThreadKey; - const inlineRightPanelOwnsTitleBar = rightPanelOpen && !shouldUsePlanSidebarSheet; const [threadPanelPresentation, setThreadPanelPresentation] = useState("inline"); const [threadPanelPopoverHandle] = useState(PopoverCreateHandle); @@ -3203,6 +3194,15 @@ export default function ChatView(props: ChatViewProps) { const cloudExecution = persistedProviderThread?.nativeMetadata?.cloudExecution; const isCloudThread = !!cloudExecution || persistedProviderThread?.driver === "kilo-cloud" || isCloudComposer; + const rightPanelOpen = !isCloudThread && rightPanelState.isOpen; + const rightPanelPresent = !isCloudThread && rightPanelPresence.present; + const rightPanelControlsInPanel = + shouldUsePlanSidebarSheet && rightPanelPresent && rightPanelOpen; + const rightPanelControlsAtRoot = rightPanelPresent && !shouldUsePlanSidebarSheet; + const canMaximizeRightPanel = rightPanelOpen && !shouldUsePlanSidebarSheet; + const rightPanelMaximized = + canMaximizeRightPanel && maximizedRightPanelThreadKey === routeThreadKey; + const inlineRightPanelOwnsTitleBar = rightPanelOpen && !shouldUsePlanSidebarSheet; const { enabled: interactionModeEnabled, interactionMode } = resolveComposerInteractionMode({ planModeEnabled: settings.planModeEnabled && !isCloudComposer, provider: activeProviderStatus, @@ -8998,6 +8998,7 @@ export default function ChatView(props: ChatViewProps) { }); if (composerRef.current?.validateProviderInput(text) === false) return; multipleTargets.push({ + requiresLocalWorktree: provider.driverKind !== "kilo-cloud", selection: createModelSelection( selection.instanceId, selection.model, @@ -9199,12 +9200,16 @@ export default function ChatView(props: ChatViewProps) { worktreePath: null, createdAt: messageCreatedAt, }, - prepareWorktree: { - projectCwd: activeProject.workspaceRoot, - baseBranch: activeThreadBranch!, - requireWorktree: true, - ...(startFromOrigin ? { startFromOrigin: true } : {}), - }, + ...(target.requiresLocalWorktree + ? { + prepareWorktree: { + projectCwd: activeProject.workspaceRoot, + baseBranch: activeThreadBranch!, + requireWorktree: true, + ...(startFromOrigin ? { startFromOrigin: true } : {}), + }, + } + : {}), runSetupScript: true, }, createdAt: messageCreatedAt, @@ -11603,7 +11608,7 @@ export default function ChatView(props: ChatViewProps) { pullRequestsAvailable={!isCloudThread && pullRequestsSurfaceAvailable} deviceAvailable={!isCloudThread && activeThreadRef !== null} > - {isCloudThread ? null : rightPanelContent} + {rightPanelContent} ) : null} {rightPanelPresent && shouldUsePlanSidebarSheet && activeThreadRef ? ( @@ -11658,7 +11663,7 @@ export default function ChatView(props: ChatViewProps) { pullRequestsAvailable={!isCloudThread && pullRequestsSurfaceAvailable} deviceAvailable={!isCloudThread && activeThreadRef !== null} > - {isCloudThread ? null : rightPanelContent} + {rightPanelContent} ) : null} From 57af33747be398d577af815d974a81c6a719fced Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Mon, 5 Oct 2026 20:02:11 +0000 Subject: [PATCH 39/44] fix(web): drop saved worktree from cloud draft bootstrap Assisted by GPT-6 in the Codex cloud harness on behalf of @thomasbrugman. --- apps/web/src/components/ChatView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 33b9038af736..9fb1a62a800e 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -9525,7 +9525,7 @@ export default function ChatView(props: ChatViewProps) { runtimeMode, interactionMode: sendInteractionMode, branch: activeThreadBranch, - worktreePath: activeThread.worktreePath, + worktreePath: isCloudComposer ? null : activeThread.worktreePath, createdAt: activeThread.createdAt, }, } From 6bb1f9bfa1cbe38f6437fa6515dbeb0786ab998e Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Mon, 5 Oct 2026 20:25:52 +0000 Subject: [PATCH 40/44] fix(web): preserve native close shortcut in cloud threads Assisted by GPT-6 in the Codex cloud harness on behalf of @thomasbrugman. --- apps/web/src/components/ChatView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 9fb1a62a800e..283162f5c6ac 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -7753,7 +7753,7 @@ export default function ChatView(props: ChatViewProps) { if (command === "rightPanel.close") { // Nothing open: leave the event alone so the shortcut keeps its // native meaning (close window on desktop, close tab in a browser). - if (!activeRightPanelSurface) return; + if (isCloudThread || !activeRightPanelSurface) return; event.preventDefault(); event.stopPropagation(); if (!event.repeat) closeRightPanelSurface(activeRightPanelSurface); From 2acd6ec14736c24ca63e7310a67fe564e28c3f77 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Tue, 6 Oct 2026 03:14:08 +0000 Subject: [PATCH 41/44] fix(kilo): keep session and cloud recovery live Restore healthy local session state before terminal events and keep exhausted cloud reattaches scheduled. Restore previously verified cloud identity without treating cached metadata as authentication. Format cloud observations and clarify initial setup. Validated with 242 focused tests, server/web/mobile typechecks, and targeted lint. Independently reviewed in the Codex cloud environment. Assisted by GPT-6 in the Codex cloud harness on behalf of @thomasbrugman. --- .../features/threads/ThreadDetailScreen.tsx | 7 +- .../Adapters/KiloAdapterV2.test.ts | 204 ++++++++++++++++++ .../Adapters/KiloAdapterV2.ts | 35 ++- .../ProviderTurnStartService.test.ts | 162 +++++++++++--- .../ProviderTurnStartService.ts | 38 +++- .../src/provider/Drivers/KiloCloudDriver.ts | 24 ++- .../provider/kilo/KiloCloudAccount.test.ts | 24 ++- .../src/provider/kilo/KiloCloudAccount.ts | 47 +++- apps/web/src/components/ChatView.tsx | 8 +- docs/user/install.md | 19 +- docs/user/providers-kilo.md | 4 + 11 files changed, 521 insertions(+), 51 deletions(-) create mode 100644 apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.test.ts diff --git a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx index d28cd805065a..7f073e1527a9 100644 --- a/apps/mobile/src/features/threads/ThreadDetailScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadDetailScreen.tsx @@ -1312,8 +1312,11 @@ export const ThreadDetailScreen = memo(function ThreadDetailScreen(props: Thread Kilo Cloud · {cloudExecution.repository} · {cloudExecution.branch} - Last observation: {cloudExecution.observedAt ?? "unavailable"}. Task:{" "} - {cloudExecutionLabel(cloudExecution.task)}.{" "} + Last observation:{" "} + {cloudExecution.observedAt + ? new Date(cloudExecution.observedAt).toLocaleString() + : "unavailable"} + . Task: {cloudExecutionLabel(cloudExecution.task)}.{" "} {cloudExecution.result ? `Result: ${cloudExecutionLabel(cloudExecution.result)}. ` : ""} diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.test.ts new file mode 100644 index 000000000000..86f29c1db595 --- /dev/null +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.test.ts @@ -0,0 +1,204 @@ +import type { Event } from "@kilocode/sdk/v2"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { + NodeId, + ProviderInstanceId, + ProviderSessionId, + RunAttemptId, + RunId, + ThreadId, +} from "@t3tools/contracts"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; +import { KiloSessionError } from "../../provider/kilo/KiloSessionClient.ts"; +import type { KiloConnection } from "../../provider/kilo/KiloRuntime.ts"; +import type * as Adapter from "../ProviderAdapter.ts"; +import * as IdAllocator from "../IdAllocator.ts"; +import * as KiloAdapter from "./KiloAdapterV2.ts"; + +it.effect.each(["reconnect", "uncertain admission", "reconnect completion"] as const)( + "clears the local session warning after %s", + (scenario) => + Effect.gen(function* () { + const disconnected = yield* Deferred.make(); + const unhealthy = yield* Deferred.make(); + const recovered = yield* Deferred.make(); + const terminal = yield* Deferred.make(); + const incoming = yield* Queue.unbounded(); + const seen: Array = []; + let subscriptions = 0; + let messageID = ""; + let completed = false; + const instanceId = ProviderInstanceId.make("kilo-session-status"); + const threadId = ThreadId.make("kilo-status-thread"); + const modelSelection = { instanceId, model: "fixture/test" }; + const runtimePolicy = { + runtimeMode: "full-access" as const, + interactionMode: "default" as const, + cwd: "/fixture", + }; + const native = { instanceId: "fixture", directory: "/fixture", sessionId: "ses_fixture" }; + const client = { + create: () => Effect.succeed(native), + read: () => Effect.succeed({}), + setPermissions: () => Effect.void, + pending: () => Effect.succeed([]), + status: () => Effect.succeed("idle"), + history: () => + Effect.succeed( + completed + ? [ + { + info: { + id: messageID, + sessionID: native.sessionId, + role: "user", + time: { created: 1 }, + }, + parts: [], + }, + { + info: { + id: "assistant", + parentID: messageID, + sessionID: native.sessionId, + role: "assistant", + time: { created: 2, completed: 3 }, + }, + parts: [], + }, + ] + : [], + ), + prompt: (_ref: unknown, input: { messageID: string }) => { + messageID = input.messageID; + return scenario === "uncertain admission" + ? Effect.fail( + new KiloSessionError({ operation: "prompt", reason: "admission_unknown" }), + ) + : Effect.void; + }, + events: (_ref: unknown, onConnected: Effect.Effect) => + Stream.unwrap( + Effect.gen(function* () { + subscriptions += 1; + yield* onConnected; + return scenario !== "uncertain admission" && subscriptions === 1 + ? Stream.fromEffect( + Deferred.await(disconnected).pipe( + Effect.andThen( + Effect.fail( + new KiloSessionError({ operation: "stream", reason: "request_failed" }), + ), + ), + ), + ) + : Stream.fromQueue(incoming); + }), + ), + } as unknown as KiloConnection["client"]; + const adapter = yield* KiloAdapter.make({ + instanceId, + continuationKey: "fixture", + cwd: "/fixture", + runtime: { + open: () => + Effect.succeed({ + client, + stop: Effect.void, + cleanup: Effect.void, + exitCode: Effect.never, + isRunning: Effect.succeed(true), + }), + }, + }); + const session = yield* adapter.openSession({ + threadId, + providerSessionId: ProviderSessionId.make("kilo-status-session"), + modelSelection, + runtimePolicy, + }); + yield* session.events.pipe( + Stream.runForEach((event) => + Effect.gen(function* () { + seen.push(event); + if (event.type === "provider_session.updated") { + if (event.providerSession.status !== "ready") + yield* Deferred.succeed(unhealthy, undefined); + else yield* Deferred.succeed(recovered, undefined); + } + if (event.type === "turn.terminal") yield* Deferred.succeed(terminal, undefined); + }), + ), + Effect.forkScoped, + ); + const providerThread = yield* session.ensureThread({ + threadId, + modelSelection, + runtimePolicy, + }); + if (scenario === "reconnect") { + yield* Deferred.succeed(disconnected, undefined); + yield* Deferred.await(unhealthy); + yield* TestClock.adjust("1 second"); + } else { + yield* session.startTurn({ + threadId, + appThread: { id: threadId }, + providerThread, + modelSelection, + runtimePolicy, + rootNodeId: NodeId.make("root"), + runId: RunId.make("run"), + attemptId: RunAttemptId.make("attempt"), + runOrdinal: 1, + providerTurnOrdinal: 1, + message: { messageId: "message", text: "Hello", attachments: [] }, + } as unknown as Adapter.ProviderAdapterV2TurnInput); + if (scenario === "reconnect completion") yield* Deferred.succeed(disconnected, undefined); + yield* Deferred.await(unhealthy); + completed = true; + if (scenario === "reconnect completion") { + yield* TestClock.adjust("1 second"); + } else { + yield* Queue.offer(incoming, { + type: "message.updated", + properties: { + info: { + id: messageID, + sessionID: native.sessionId, + role: "user", + time: { created: 1 }, + }, + }, + } as Event); + yield* Queue.offer(incoming, { + id: "idle", + type: "session.idle", + properties: { sessionID: native.sessionId }, + }); + } + yield* Deferred.await(terminal); + assert.isBelow( + seen.findIndex( + (event) => + event.type === "provider_session.updated" && event.providerSession.status === "ready", + ), + seen.findIndex((event) => event.type === "turn.terminal"), + ); + } + yield* Deferred.await(recovered); + const updates = seen.filter((event) => event.type === "provider_session.updated"); + assert.equal(updates.at(-1)?.providerSession.status, "ready"); + assert.equal(updates.at(-1)?.providerSession.lastError, null); + assert.equal( + updates[0]?.providerSession.status, + scenario === "uncertain admission" ? "waiting" : "error", + ); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(IdAllocator.layer, NodeServices.layer))), +); diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts index b6effad78ed4..53fd044217e0 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.ts @@ -6,6 +6,7 @@ import { type OrchestrationV2Subagent, type ProviderInstanceId, type OrchestrationV2ProviderCapabilities, + type OrchestrationV2ProviderSession, type OrchestrationV2ProviderThread, type OrchestrationV2ProviderTurn, type OrchestrationV2ConversationMessage, @@ -448,7 +449,7 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { const connection = yield* wire(options.runtime.open(directory)); const client = connection.client; const now = yield* DateTime.now; - const session = { + let session: OrchestrationV2ProviderSession = { id: input.providerSessionId, driver: KILO_PROVIDER, providerInstanceId: options.instanceId, @@ -465,6 +466,7 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { const items = new Map(); const emit = (event: Adapter.ProviderAdapterV2Event) => Effect.suspend(() => { + if (event.type === "provider_session.updated") session = event.providerSession; if (event.type === "node.updated") nodes.set(event.node.id, event.node); if (event.type === "turn_item.updated") items.set(event.turnItem.id, event.turnItem); return Queue.offer(events, event).pipe(Effect.asVoid); @@ -601,6 +603,17 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { }); } active = undefined; + if (session.status !== "ready" && status !== "interrupted" && (yield* connection.isRunning)) + yield* emit({ + type: "provider_session.updated", + driver: KILO_PROVIDER, + providerSession: { + ...session, + status: "ready", + lastError: null, + updatedAt: completedAt, + }, + }); yield* emit({ type: "provider_turn.updated", driver: KILO_PROVIDER, @@ -1277,6 +1290,26 @@ export const make = Effect.fn("KiloAdapterV2.make")(function* (options: { : Effect.void, ), ), + Effect.andThen( + Effect.gen(function* () { + if ( + session.status === "ready" || + (session.status === "waiting" && active && !active.admitted) || + !(yield* connection.isRunning) + ) + return; + yield* emit({ + type: "provider_session.updated", + driver: KILO_PROVIDER, + providerSession: { + ...session, + status: "ready", + lastError: null, + updatedAt: yield* DateTime.now, + }, + }); + }), + ), ), true, ) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 06761c13b3fd..01d41799dee8 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -29,6 +29,7 @@ import * as ProjectService from "../project/ProjectService.ts"; import * as ProviderAuthService from "../provider/ProviderAuthService.ts"; import * as ContextHandoffService from "./ContextHandoffService.ts"; import * as EventSink from "./EventSink.ts"; +import type * as EffectOutbox from "./EffectOutbox.ts"; import * as IdAllocator from "./IdAllocator.ts"; import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; @@ -171,6 +172,7 @@ function makeLocalCommandHarness(input: { readonly writeFailure?: unknown; /** Loads the thread and starts the run, then fails every later state read. */ readonly failReadsAfterRunning?: boolean; + readonly reattachStartFailure?: "starting" | "interrupted"; }) { const now = DateTime.makeUnsafe("2026-09-04T12:00:00Z"); const threadId = ThreadId.make("thread-native-account-command"); @@ -356,6 +358,7 @@ function makeLocalCommandHarness(input: { }; } const events: Array = []; + const effects: Array = []; const interruptRun = () => { projection = { ...projection, @@ -414,7 +417,7 @@ function makeLocalCommandHarness(input: { ), ), ) - : input.failReadsAfterRunning === true + : input.failReadsAfterRunning === true || input.reattachStartFailure !== undefined ? Effect.succeed({ driver: providerThread.driver, providerSession: { @@ -434,11 +437,34 @@ function makeLocalCommandHarness(input: { : Effect.die("A local command must not open a native session."), ); const startRootRun = vi.fn< - (input: RunExecutionService.RunExecutionServiceV2StartRootRunInput) => Effect.Effect - >(() => - input.failReadsAfterRunning === true - ? Effect.void - : Effect.die("A local command must not start a native turn."), + ( + input: RunExecutionService.RunExecutionServiceV2StartRootRunInput, + ) => Effect.Effect + >((startInput) => + input.reattachStartFailure !== undefined + ? Effect.sync(() => { + projection = { + ...projection, + runs: projection.runs.map((candidate) => + candidate.id === runId + ? { ...candidate, status: input.reattachStartFailure! } + : candidate, + ), + }; + }).pipe( + Effect.andThen( + Effect.fail( + new RunExecutionService.RunExecutionStartError({ + commandId: startInput.commandId, + runId, + cause: "remote journal unavailable", + }), + ), + ), + ) + : input.failReadsAfterRunning === true + ? Effect.void + : Effect.die("A local command must not start a native turn."), ); const failReadIfRunning = Effect.suspend(() => input.failReadsAfterRunning === true && @@ -459,29 +485,36 @@ function makeLocalCommandHarness(input: { }), ), ); - const writeIfRunCurrent = vi.fn(({ events: incoming, activeAttemptId, expectedStatus }) => - "writeFailure" in input - ? Effect.fail( - new EventSink.EventSinkWriteError({ - eventCount: incoming.length, - cause: input.writeFailure, - }), - ) - : Effect.sync(() => { - const current = projection.runs.find((candidate) => candidate.id === runId); - const committed = - current !== undefined && - current.activeAttemptId === activeAttemptId && - current.status === expectedStatus; - if (committed) { - for (const event of incoming) { - expect(isDomainEvent(event)).toBe(true); - events.push(event); - projection = ProjectionStore.applyToProjection(projection, event); + const writeIfRunCurrent = vi.fn( + ({ + events: incoming, + effects: pending = [], + activeAttemptId, + expectedStatus, + }: Parameters[0]) => + "writeFailure" in input + ? Effect.fail( + new EventSink.EventSinkWriteError({ + eventCount: incoming.length, + cause: input.writeFailure, + }), + ) + : Effect.sync(() => { + const current = projection.runs.find((candidate) => candidate.id === runId); + const committed = + current !== undefined && + current.activeAttemptId === activeAttemptId && + current.status === expectedStatus; + if (committed) { + effects.push(...pending); + for (const event of incoming) { + expect(isDomainEvent(event)).toBe(true); + events.push(event); + projection = ProjectionStore.applyToProjection(projection, event); + } } - } - return { committed, storedEvents: [] }; - }), + return { committed, storedEvents: [] }; + }), ); const layer = ProviderTurnStart.layer.pipe( Layer.provide( @@ -536,6 +569,8 @@ function makeLocalCommandHarness(input: { startRootRun, tryHandlePromptCommand, events, + effects, + interruptRun, oldInstanceId, newInstanceId, attemptId, @@ -543,6 +578,15 @@ function makeLocalCommandHarness(input: { start: Effect.gen(function* () { yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({ threadId, runId }); }).pipe(Effect.provide(layer)), + reattach: (willRetry = false) => + Effect.gen(function* () { + yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({ + threadId, + runId, + reattach: true, + willRetry, + }); + }).pipe(Effect.provide(layer)), startWithRetry: Effect.gen(function* () { yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({ threadId, @@ -855,3 +899,65 @@ for (const previousMessages of [[], ["/compact", " /COMPACT "]]) { }), ); } + +for (const failure of ["openFailure", "ensureThreadFailure"] as const) { + effectIt.effect(`reschedules an exhausted cloud reattach after ${failure}`, () => + Effect.gen(function* () { + const harness = makeLocalCommandHarness({ + text: "Continue", + [failure]: new Error("offline"), + }); + const now = yield* DateTime.now; + yield* harness.reattach(true).pipe(Effect.flip); + expect(harness.effects).toEqual([]); + yield* harness.reattach(); + yield* harness.reattach(); + expect(harness.effects).toHaveLength(2); + expect(harness.effects[0]?.id).not.toBe(harness.effects[1]?.id); + for (const effect of harness.effects) { + expect(effect.request).toEqual({ + type: "provider-turn.reattach", + runId: harness.projection().runs.at(-1)?.id, + }); + expect(effect.commandId).toBe( + `command:cloud-reattach:${harness.projection().runs.at(-1)?.id}:${harness.attemptId}`, + ); + expect(DateTime.toEpochMillis(effect.availableAt!)).toBe( + DateTime.toEpochMillis(now) + 30_000, + ); + } + expect(harness.projection().runs.at(-1)?.status).toBe("starting"); + expect(harness.startRootRun).not.toHaveBeenCalled(); + harness.interruptRun(); + yield* harness.reattach(); + expect(harness.effects).toHaveLength(2); + }), + ); +} + +effectIt.effect("returns a persistence failure when a cloud reattach cannot be rescheduled", () => + Effect.gen(function* () { + const harness = makeLocalCommandHarness({ + text: "Continue", + openFailure: new Error("offline"), + writeFailure: new Error("database unavailable"), + }); + const failure = yield* harness.reattach().pipe(Effect.flip); + expect(failure._tag).toBe("ProviderTurnStartError"); + expect(harness.effects).toEqual([]); + expect(harness.projection().runs.at(-1)?.status).toBe("starting"); + }), +); + +for (const status of ["starting", "interrupted"] as const) { + effectIt.effect(`guards exhausted reattach execution failure in ${status}`, () => + Effect.gen(function* () { + const harness = makeLocalCommandHarness({ text: "Continue", reattachStartFailure: status }); + yield* harness.reattach(); + expect(harness.startRootRun).toHaveBeenCalledOnce(); + expect(harness.startRootRun.mock.calls[0]?.[0].reattach).toBe(true); + expect(harness.projection().runs.at(-1)?.status).toBe(status); + expect(harness.effects).toHaveLength(status === "starting" ? 1 : 0); + }), + ); +} diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 38469ff67e71..373eef52f4db 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -542,6 +542,26 @@ export const layer: Layer.Layer< inheritedBackgroundTurnItems, }); const { isCurrentAttemptInStatus } = runControls; + const rescheduleReattach = Effect.gen(function* () { + const now = yield* DateTime.now; + const commandId = CommandId.make(`command:cloud-reattach:${run.id}:${attempt.id}`); + yield* eventSink.writeIfRunCurrent({ + threadId: projection.thread.id, + runId: run.id, + activeAttemptId: attempt.id, + expectedStatus: "starting", + events: [], + effects: [ + { + id: `effect:cloud-reattach:${yield* idAllocator.allocate.event({ threadId: projection.thread.id, commandId })}`, + commandId, + threadId: projection.thread.id, + request: { type: "provider-turn.reattach", runId: run.id }, + availableAt: DateTime.add(now, { seconds: 30 }), + }, + ], + }); + }); const resolvedRuntimePolicy = yield* runtimePolicy.resolve({ thread: projection.thread, @@ -604,8 +624,8 @@ export const layer: Layer.Layer< }); }); if (sessionResult._tag === "Failure") { - if (input.willRetry === true || input.reattach === true) - return yield* sessionResult.failure; + if (input.willRetry === true) return yield* sessionResult.failure; + if (input.reattach === true) return yield* rescheduleReattach; yield* settleStartFailure({ signal: "provider-session-open-failure", title: "Provider session failed to open", @@ -622,7 +642,11 @@ export const layer: Layer.Layer< Effect.gen(function* () { const loaded = yield* Effect.result(load); if (loaded._tag === "Success") return loaded.success; - if (input.willRetry === true || input.reattach === true) return yield* loaded.failure; + if (input.willRetry === true) return yield* loaded.failure; + if (input.reattach === true) { + yield* rescheduleReattach; + return undefined; + } yield* settleStartFailure({ signal: "provider-thread-load-failure", title: "Provider turn failed to start", @@ -1235,7 +1259,7 @@ export const layer: Layer.Layer< !noteContinuation ? session : makeDeliverySession(session, startWithHandoffs); - yield* runExecution.startRootRun({ + const execution = runExecution.startRootRun({ ...(input.reattach ? { reattach: true } : {}), commandId: CommandId.make(`command:effect:provider-turn.start:${run.id}`), appThread: projection.thread, @@ -1287,6 +1311,12 @@ export const layer: Layer.Layer< modelSelection: run.modelSelection, runtimePolicy: resolvedRuntimePolicy, }); + yield* execution.pipe( + Effect.catchIf( + () => input.reattach === true && input.willRetry !== true, + () => rescheduleReattach, + ), + ); }); return ProviderTurnStartServiceV2.of({ diff --git a/apps/server/src/provider/Drivers/KiloCloudDriver.ts b/apps/server/src/provider/Drivers/KiloCloudDriver.ts index 4c0477c728d4..37d486ec05be 100644 --- a/apps/server/src/provider/Drivers/KiloCloudDriver.ts +++ b/apps/server/src/provider/Drivers/KiloCloudDriver.ts @@ -36,8 +36,20 @@ export const KiloCloudDriver: ProviderDriver diff --git a/apps/server/src/provider/kilo/KiloCloudAccount.test.ts b/apps/server/src/provider/kilo/KiloCloudAccount.test.ts index eb718797c119..bab801dcaf56 100644 --- a/apps/server/src/provider/kilo/KiloCloudAccount.test.ts +++ b/apps/server/src/provider/kilo/KiloCloudAccount.test.ts @@ -13,6 +13,7 @@ it.live("separates credential rejection, account support and temporary profile f Effect.gen(function* () { let responseStatus = 503; let personal = true; + let requests = 0; const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-cloud-account-" }); yield* fs.makeDirectory(`${root}/data/kilo`, { recursive: true }); @@ -22,6 +23,7 @@ it.live("separates credential rejection, account support and temporary profile f const server = yield* Effect.acquireRelease( Effect.promise(async () => { const server = NodeHttp.createServer((req, res) => { + requests += 1; res.writeHead(responseStatus, { "content-type": "application/json" }); res.end( encode({ @@ -45,7 +47,9 @@ it.live("separates credential rejection, account support and temporary profile f ); const address = server.address(); if (!address || typeof address === "string") return yield* Effect.die("No fixture address"); - const account = yield* Account.make(root, `http://127.0.0.1:${address.port}`); + const origin = `http://127.0.0.1:${address.port}`; + const identityPath = `${root}/state/account.json`; + const account = yield* Account.make(root, origin, identityPath); assert.equal((yield* account.load.pipe(Effect.flip)).reason, "invalid_response"); responseStatus = 401; assert.equal((yield* account.load.pipe(Effect.flip)).reason, "rejected"); @@ -54,9 +58,27 @@ it.live("separates credential rejection, account support and temporary profile f assert.equal((yield* account.load.pipe(Effect.flip)).reason, "unsupported"); personal = true; assert.equal((yield* account.load).accountId, "a"); + const savedIdentity = yield* fs.readFileString(identityPath); + assert.isFalse(savedIdentity.includes("synthetic-a")); + responseStatus = 503; + const restarted = yield* Account.make(root, origin, identityPath); + const requestCount = requests; + const restored = yield* restarted.restore; + assert.equal(restored.accountId, "a"); + assert.isFalse(restored.verified); + assert.equal(requests, requestCount); + assert.equal((yield* restarted.load.pipe(Effect.flip)).reason, "invalid_response"); + responseStatus = 401; + assert.equal((yield* restarted.load.pipe(Effect.flip)).reason, "rejected"); + responseStatus = 200; + assert.equal((yield* restarted.load).accountId, "a"); yield* write("synthetic-b"); + responseStatus = 503; + assert.equal((yield* restarted.restore.pipe(Effect.flip)).reason, "invalid_response"); + responseStatus = 200; assert.equal((yield* account.load).accountId, "b"); yield* fs.writeFileString(`${root}/data/kilo/auth.json`, "malformed"); assert.equal((yield* account.load.pipe(Effect.flip)).reason, "rejected"); + assert.equal((yield* restarted.restore.pipe(Effect.flip)).reason, "rejected"); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); diff --git a/apps/server/src/provider/kilo/KiloCloudAccount.ts b/apps/server/src/provider/kilo/KiloCloudAccount.ts index f0716d3169b7..5408e06e683e 100644 --- a/apps/server/src/provider/kilo/KiloCloudAccount.ts +++ b/apps/server/src/provider/kilo/KiloCloudAccount.ts @@ -1,4 +1,6 @@ import * as Effect from "effect/Effect"; +import * as Crypto from "effect/Crypto"; +import { Hex } from "effect/encoding"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Redacted from "effect/Redacted"; @@ -21,18 +23,26 @@ const Profile = Schema.Struct({ const decodeAuth = Schema.decodeUnknownEffect(Schema.fromJsonString(Auth)); const decodeProfile = Schema.decodeUnknownEffect(Profile); +const Identity = Schema.Struct({ + accountId: Schema.NonEmptyString, + tokenHash: Schema.NonEmptyString, +}); +const decodeIdentity = Schema.decodeUnknownEffect(Schema.fromJsonString(Identity)); +const encodeIdentity = Schema.encodeSync(Schema.fromJsonString(Identity)); /** Reads only the selected official CLI profile. Login and token refresh remain Kilo's job. */ export const make = Effect.fn("KiloCloudAccount.make")(function* ( profileDirectory: string, origin = "https://app.kilo.ai", + identityPath?: string, ) { if (origin !== "https://app.kilo.ai" && !/^http:\/\/127\.0\.0\.1:\d+$/.test(origin)) return yield* new KiloCloudError({ operation: "authentication", reason: "rejected" }); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; let cached: { token: Redacted.Redacted; accountId: string } | undefined; - const load = Effect.gen(function* () { + const readToken = Effect.gen(function* () { const saved = yield* decodeAuth( yield* fs .readFileString(path.join(profileDirectory, "data", "kilo", "auth.json")) @@ -45,6 +55,12 @@ export const make = Effect.fn("KiloCloudAccount.make")(function* ( Effect.mapError(() => new KiloCloudError({ operation: "credentials", reason: "rejected" })), ); const token = saved.kilo.type === "oauth" ? saved.kilo.access : saved.kilo.key; + return token; + }); + const tokenHash = (token: string) => + crypto.digest("SHA-256", new TextEncoder().encode(token)).pipe(Effect.map(Hex.encode)); + const load = Effect.gen(function* () { + const token = yield* readToken; if (cached && Redacted.value(cached.token) === token) return cached; const client = yield* HttpClient.HttpClient; const response = yield* client.execute( @@ -62,6 +78,15 @@ export const make = Effect.fn("KiloCloudAccount.make")(function* ( if (!profile.hasPersonalAccount) return yield* new KiloCloudError({ operation: "personal-account", reason: "unsupported" }); cached = { accountId: profile.user.id, token: Redacted.make(token) }; + if (identityPath) { + const identity = encodeIdentity({ + accountId: cached.accountId, + tokenHash: yield* tokenHash(token), + }); + yield* fs + .makeDirectory(path.dirname(identityPath), { recursive: true }) + .pipe(Effect.andThen(fs.writeFileString(identityPath, identity)), Effect.ignore); + } return cached; }).pipe( Effect.scoped, @@ -74,5 +99,23 @@ export const make = Effect.fn("KiloCloudAccount.make")(function* ( : new KiloCloudError({ operation: "authentication", reason: "invalid_response" }), ), ); - return { load }; + // This restores only the immutable binding. Requests still verify through load. + const restore = Effect.gen(function* () { + if (identityPath) { + const token = yield* readToken; + const prior = yield* fs + .readFileString(identityPath) + .pipe(Effect.flatMap(decodeIdentity), Effect.option); + if (prior._tag === "Some" && prior.value.tokenHash === (yield* tokenHash(token))) + return { accountId: prior.value.accountId, token: Redacted.make(token), verified: false }; + } + return { ...(yield* load), verified: true }; + }).pipe( + Effect.mapError((cause) => + isKiloCloudError(cause) + ? cause + : new KiloCloudError({ operation: "authentication", reason: "invalid_response" }), + ), + ); + return { load, restore }; }); diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 6a5494c1790f..9c5a1c0339c7 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -1,4 +1,5 @@ import { cloudExecutionLabel } from "@t3tools/client-runtime/cloudExecutionLabels"; +import { formatChatTimestampTooltip } from "../timestampFormat"; import { ChatCanvas } from "./chat/ChatCanvas"; import { usageLimitRecoveryBannerItem } from "./chat/UsageLimitRecoveryBanner"; import { @@ -11067,8 +11068,11 @@ export default function ChatView(props: ChatViewProps) {

{cloudExecution ? (

- Last observation: {cloudExecution.observedAt ?? "unavailable"}. Task:{" "} - {cloudExecutionLabel(cloudExecution.task)}.{" "} + Last observation:{" "} + {cloudExecution.observedAt + ? formatChatTimestampTooltip(cloudExecution.observedAt, timestampFormat) + : "unavailable"} + . Task: {cloudExecutionLabel(cloudExecution.task)}.{" "} {cloudExecution.result ? `Result: ${cloudExecutionLabel(cloudExecution.result)}. ` : ""} diff --git a/docs/user/install.md b/docs/user/install.md index e3347decab1f..76e7682c5fe1 100644 --- a/docs/user/install.md +++ b/docs/user/install.md @@ -125,15 +125,16 @@ and enable the provider you want. Installation, login, and configuration belong to that environment's machine, even when you connect from a phone or another computer. -| Provider | Install and authenticate | -| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Codex | [Connect with ChatGPT](./providers-codex.md#connect-with-chatgpt), or install [Codex CLI](https://developers.openai.com/codex/cli) and run `codex login`. | -| Claude | Install [Claude Code](https://claude.com/product/claude-code), then run `claude auth login`. | -| Cursor | Install [Cursor CLI](https://cursor.com/cli), then run `agent login`. | -| Grok Build | Install [Grok Build CLI](https://x.ai/cli), then run `grok login`. | -| OpenCode | Install [OpenCode](https://opencode.ai), then run `opencode auth login`. | -| Antigravity | Install and sign in with Google from T3 Code's provider settings. | -| Pi | Install [Pi](https://pi.dev), then run `pi` once to finish its login or API-key setup. | +| Provider | Install and authenticate | +| ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Codex | [Connect with ChatGPT](./providers-codex.md#connect-with-chatgpt), or install [Codex CLI](https://developers.openai.com/codex/cli) and run `codex login`. | +| Claude | Install [Claude Code](https://claude.com/product/claude-code), then run `claude auth login`. | +| Cursor | Install [Cursor CLI](https://cursor.com/cli), then run `agent login`. | +| Grok Build | Install [Grok Build CLI](https://x.ai/cli), then run `grok login`. | +| OpenCode | Install [OpenCode](https://opencode.ai), then run `opencode auth login`. | +| Antigravity | Install and sign in with Google from T3 Code's provider settings. | +| Pi | Install [Pi](https://pi.dev), then run `pi` once to finish its login or API-key setup. | +| Kilo / Kilo Cloud | Install Kilo CLI 7.8.3 and sign in with the official CLI. See [Kilo setup](./providers-kilo.md) for local and cloud requirements. | Provider CLIs must be on the server's `PATH`. If T3 Code cannot find one, set its **Binary path** in provider settings, especially when using a version manager. diff --git a/docs/user/providers-kilo.md b/docs/user/providers-kilo.md index c4e516c07caa..068083b82f0f 100644 --- a/docs/user/providers-kilo.md +++ b/docs/user/providers-kilo.md @@ -30,6 +30,10 @@ Add **Kilo Cloud** with a profile directory signed in through the official Kilo CLI, a GitHub repository Kilo can access, its branch, and a model. Then turn on **Allow paid cloud execution**. Personal accounts are supported. +First setup and changed login credentials require online account verification. +If that check is unavailable during setup, retry by reconfiguring the provider +or restarting T3 Code once Kilo is reachable. + Prompts and the selected repository go to Kilo. T3 Code never uploads local files or uncommitted changes, and each cloud thread works in its own remote worktree. Start cloud threads at the project root. Local attachments, terminals, Git From 43fce221e2273a6ff172a518aefff138f4977ffb Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Tue, 6 Oct 2026 03:32:35 +0000 Subject: [PATCH 42/44] fix(mobile): gate Kilo terminal routes on workspace availability Require the shared local workspace state for terminal navigation and Android menu actions. This keeps unresolved provider bindings from exposing local terminals. Verified with four workspace classification tests, mobile typecheck, targeted lint, and independent review. Assisted by GPT-6 in the Codex cloud harness on behalf of @thomasbrugman. --- .../features/threads/ThreadRouteScreen.tsx | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/apps/mobile/src/features/threads/ThreadRouteScreen.tsx b/apps/mobile/src/features/threads/ThreadRouteScreen.tsx index 14cfda6f8dc2..a1d74802ae09 100644 --- a/apps/mobile/src/features/threads/ThreadRouteScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadRouteScreen.tsx @@ -722,7 +722,12 @@ function ThreadRouteContent( hasWorkspaceRoot: Boolean(selectedThreadProject?.workspaceRoot), }); - if (isCloudThread || !selectedThread || !selectedThreadProject?.workspaceRoot) { + if ( + isCloudThread || + !localWorkspaceEnabled || + !selectedThread || + !selectedThreadProject?.workspaceRoot + ) { return; } @@ -732,7 +737,13 @@ function ThreadRouteContent( ...(nextTerminalId ? { terminalId: nextTerminalId } : {}), }); }, - [isCloudThread, navigation, selectedThread, selectedThreadProject?.workspaceRoot], + [ + isCloudThread, + localWorkspaceEnabled, + navigation, + selectedThread, + selectedThreadProject?.workspaceRoot, + ], ); const handleOpenNewTerminal = useCallback(() => { @@ -742,7 +753,12 @@ function ThreadRouteContent( listedTerminalIds: terminalMenuSessions.map((session) => session.terminalId), }); - if (isCloudThread || !selectedThread || !selectedThreadProject?.workspaceRoot) { + if ( + isCloudThread || + !localWorkspaceEnabled || + !selectedThread || + !selectedThreadProject?.workspaceRoot + ) { return; } @@ -756,6 +772,7 @@ function ThreadRouteContent( }); }, [ isCloudThread, + localWorkspaceEnabled, navigation, selectedThread, selectedThreadProject?.workspaceRoot, @@ -859,7 +876,8 @@ function ThreadRouteContent( currentBranch: selectedThread?.branch ?? null, gitStatus: gitStatus.data, gitOperationLabel: gitState.gitOperationLabel, - canOpenTerminal: !isCloudThread && Boolean(selectedThreadProject?.workspaceRoot), + canOpenTerminal: + !isCloudThread && localWorkspaceEnabled && Boolean(selectedThreadProject?.workspaceRoot), canOpenFiles: !isCloudThread && Boolean(selectedThreadProject?.workspaceRoot), projectScripts: selectedThreadProject ? resolveProjectScripts( @@ -1134,7 +1152,9 @@ function ThreadRouteContent( usesNativeHeaderGlass={usesNativeHeaderGlass} gitControls={threadGitControlProps} hasThreadCwd={!isCloudThread && selectedThreadCwd !== null} - hasWorkspaceRoot={!isCloudThread && Boolean(selectedThreadProject?.workspaceRoot)} + hasWorkspaceRoot={ + !isCloudThread && localWorkspaceEnabled && Boolean(selectedThreadProject?.workspaceRoot) + } fileInspectorSupported={!isCloudThread && fileInspector.supported} inspectorMode={inspectorMode} onToggleInspector={handleToggleInspector} From 75f941c608e6ce46a979280d42d4ae30557d2aa3 Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Tue, 6 Oct 2026 10:16:18 +0000 Subject: [PATCH 43/44] fix(kilo): address workspace actions and service review findings --- .../features/threads/ThreadRouteScreen.tsx | 15 ++++---- .../Adapters/KiloAdapterV2.live.test.ts | 4 ++- .../server/src/provider/Drivers/KiloDriver.ts | 4 ++- .../textGeneration/KiloTextGeneration.test.ts | 34 +++++++++++++++++++ .../src/textGeneration/KiloTextGeneration.ts | 11 +++--- 5 files changed, 53 insertions(+), 15 deletions(-) create mode 100644 apps/server/src/textGeneration/KiloTextGeneration.test.ts diff --git a/apps/mobile/src/features/threads/ThreadRouteScreen.tsx b/apps/mobile/src/features/threads/ThreadRouteScreen.tsx index 19e03c005d62..e175a71a2577 100644 --- a/apps/mobile/src/features/threads/ThreadRouteScreen.tsx +++ b/apps/mobile/src/features/threads/ThreadRouteScreen.tsx @@ -124,11 +124,13 @@ function ThreadHeader( onPress: () => onOpenTerminal(null), }); } - actions.push({ - accessibilityLabel: "Open git controls", - icon: "point.topleft.down.curvedto.point.bottomright.up", - onPress: props.onOpenGitInspector, - }); + if (props.hasWorkspaceRoot) { + actions.push({ + accessibilityLabel: "Open git controls", + icon: "point.topleft.down.curvedto.point.bottomright.up", + onPress: props.onOpenGitInspector, + }); + } if (onMergeBack) { actions.push({ accessibilityLabel: "Merge back to source", @@ -547,7 +549,7 @@ function ThreadRouteContent( const gitActionProgress = useGitActionProgress(gitActionProgressTarget); const handleOpenGitInspector = useCallback(() => { - if (isCloudThread) return; + if (isCloudThread || !localWorkspaceEnabled) return; if (!fileInspector.supported) { if (selectedThread === null) { return; @@ -563,6 +565,7 @@ function ThreadRouteContent( }, [ fileInspector.supported, isCloudThread, + localWorkspaceEnabled, navigation, routeThreadIdentity, selectedThread, diff --git a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts index 6fa3a113bea3..71340169e39f 100644 --- a/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/KiloAdapterV2.live.test.ts @@ -588,7 +588,9 @@ require('node:readline').createInterface({input:process.stdin}).on('line',line=> assert.isFalse(seen.slice(before).some((event) => event.type === "app_thread.created")); } model.control.mode = "json"; - const textGeneration = KiloTextGeneration.make(runtime); + const textGeneration = yield* KiloTextGeneration.make().pipe( + Effect.provideService(KiloRuntime.KiloRuntime, runtime), + ); const generated = yield* textGeneration.generateThreadTitle({ cwd: root, modelSelection, diff --git a/apps/server/src/provider/Drivers/KiloDriver.ts b/apps/server/src/provider/Drivers/KiloDriver.ts index 0a8f87fde30e..7bbd6dc26d12 100644 --- a/apps/server/src/provider/Drivers/KiloDriver.ts +++ b/apps/server/src/provider/Drivers/KiloDriver.ts @@ -269,7 +269,9 @@ export const KiloDriver: ProviderDriver = { }), ); if (input.enabled) yield* refresh.pipe(Effect.forkScoped); - const textGeneration = KiloTextGeneration.make(runtime, server.attachmentsDir); + const textGeneration = yield* KiloTextGeneration.make(server.attachmentsDir).pipe( + Effect.provideService(KiloRuntime.KiloRuntime, runtime), + ); return { instanceId: input.instanceId, driverKind: kind, diff --git a/apps/server/src/textGeneration/KiloTextGeneration.test.ts b/apps/server/src/textGeneration/KiloTextGeneration.test.ts new file mode 100644 index 000000000000..7360876ff7d8 --- /dev/null +++ b/apps/server/src/textGeneration/KiloTextGeneration.test.ts @@ -0,0 +1,34 @@ +import { assert, it } from "@effect/vitest"; +import { ProviderInstanceId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; + +import * as KiloRuntime from "../provider/kilo/KiloRuntime.ts"; +import * as KiloTextGeneration from "./KiloTextGeneration.ts"; + +it.effect("keeps runtime diagnostics in the cause of a bounded text-generation error", () => + Effect.gen(function* () { + const cause = new KiloRuntime.KiloRuntimeError({ + operation: "open", + detail: "Private runtime diagnostic", + }); + const textGeneration = yield* KiloTextGeneration.make().pipe( + Effect.provideService(KiloRuntime.KiloRuntime, { + open: () => Effect.fail(cause), + }), + ); + const error = yield* textGeneration + .generateThreadTitle({ + cwd: "/workspace/project", + modelSelection: { + instanceId: ProviderInstanceId.make("kilo"), + model: "provider/model", + }, + message: "Name this thread", + }) + .pipe(Effect.flip); + + assert.equal(error.operation, "generateThreadTitle"); + assert.equal(error.detail, "Kilo text generation failed. The request was not retried."); + assert.strictEqual(error.cause, cause); + }), +); diff --git a/apps/server/src/textGeneration/KiloTextGeneration.ts b/apps/server/src/textGeneration/KiloTextGeneration.ts index abbde75bd729..496edc06e182 100644 --- a/apps/server/src/textGeneration/KiloTextGeneration.ts +++ b/apps/server/src/textGeneration/KiloTextGeneration.ts @@ -7,12 +7,11 @@ import { toOpenCodeFileParts } from "../provider/opencodeRuntime.ts"; import * as KiloRuntime from "../provider/kilo/KiloRuntime.ts"; import * as TextGenerationOperations from "./TextGenerationOperations.ts"; -const isKiloRuntimeError = Schema.is(KiloRuntime.KiloRuntimeError); - const isTextGenerationError = Schema.is(TextGenerationError); /** Only prompt construction is shared. Protocol, credentials and lifetime belong to Kilo. */ -export function make(runtime: KiloRuntime.KiloRuntime["Service"], attachmentsDir?: string) { +export const make = Effect.fn("KiloTextGeneration.make")(function* (attachmentsDir?: string) { + const runtime = yield* KiloRuntime.KiloRuntime; const run: TextGenerationOperations.Runner = (input) => Effect.gen(function* () { const separator = input.modelSelection.model.indexOf("/"); @@ -60,12 +59,10 @@ export function make(runtime: KiloRuntime.KiloRuntime["Service"], attachmentsDir ? cause : new TextGenerationError({ operation: input.operation, - detail: isKiloRuntimeError(cause) - ? cause.message - : "Kilo text generation failed. The request was not retried.", + detail: "Kilo text generation failed. The request was not retried.", cause, }), ), ); return TextGenerationOperations.fromRunner("KiloTextGeneration", run); -} +}); From ef21baa7fbcd844caf4e5f70e8f1ba000ef12eba Mon Sep 17 00:00:00 2001 From: Thomas Brugman Date: Tue, 6 Oct 2026 15:53:08 +0000 Subject: [PATCH 44/44] fix(kilo): download cloud file previews --- apps/web/src/components/ChatView.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 44434124e0e3..2b3faf97aa0a 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -11239,7 +11239,13 @@ export default function ChatView(props: ChatViewProps) { : {})} isRevertingCheckpoint={isRevertingCheckpoint} onImageExpand={onExpandTimelineImage} - onFileOpen={paintOnlyDisplayedTimeline ? noopHeldAttachment : openFileAttachment} + onFileOpen={ + paintOnlyDisplayedTimeline + ? noopHeldAttachment + : isCloudThread + ? downloadFileAttachment + : openFileAttachment + } onFileDownload={ paintOnlyDisplayedTimeline ? noopHeldAttachment : downloadFileAttachment }