diff --git a/ReadMe.md b/ReadMe.md index 227468e..12ba08a 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -48,20 +48,21 @@ npx skynot [options] The following command‑line flags are available: -| Flag | Alias | Description | -|--------------|--------|-----------------------------------------------------------------------------------| -|`--help` | `-h` | Show the help message with all available options. | -|`--auth` | `-a` | Prompt for AI model's credentials to add env var to script or cook auth.json file.| -|`--extensions`| `-e` | DEPRECATED: Use `spi install ` instead, after install. | -|`--git ["id"]`| `-g[i]`| Set git `user.name`/`user.email` for `aidev`. No arg: copies from current user. | -| | | With arg (e.g. `"Name Surname "`): uses that instead. | -|`--npm` | `-n` | Install Pi using npm instead of tarball (likely to be slower though). | -|`--paranoid` | `-p` | Refrain from caching the sudo password; ask for it every time it is needed. | -|`--ssh` | `-s` | Copy SSH keys to the `aidev` user for git+ssh (& add GitHub to `known_hosts`). | -|`--update` | `-u` | Wipe any previous existing install of Pi and reinstall, to get the latest version.| -|`--verbose` | `-v` | Show more output from install commands (useful for debugging/low-bandwidth). | -|`--version` | `-V` | Output the version number. | -|`--destroy` |`--BURN`| Delete the `aidev` user, all its data (in `$HOME`), and the `aiteam` group. | +| Flag | Alias | Description | +|----------------|--------|-----------------------------------------------------------------------------------| +|`--help` | `-h` | Show the help message with all available options. | +|`--auth` | `-a` | Prompt for AI model's credentials to add env var to script or cook auth.json file.| +|`--extensions` | `-e` | DEPRECATED: Use `spi install ` instead, after install. | +|`--git ["id"]` | `-g[i]`| Set git `user.name`/`user.email` for `aidev`. No arg: copies from current user. | +| | | With arg (e.g. `"Name Surname "`): uses that instead. | +|`--npm` | `-n` | Install Pi using npm instead of tarball (likely to be slower though). | +|`--context-lens`| `-c` | Install context-lens and wrapper script `cpi` for launching pi with context-lens. | +|`--paranoid` | `-p` | Refrain from caching the sudo password; ask for it every time it is needed. | +|`--ssh` | `-s` | Copy SSH keys to the `aidev` user for git+ssh (& add GitHub to `known_hosts`). | +|`--update` | `-u` | Wipe any previous existing install of Pi and reinstall, to get the latest version.| +|`--verbose` | `-v` | Show more output from install commands (useful for debugging/low-bandwidth). | +|`--version` | `-V` | Output the version number. | +|`--destroy` |`--BURN`| Delete the `aidev` user, all its data (in `$HOME`), and the `aiteam` group. | Please note, `-u` would technically not wipe or reinstall extensions, as they normally live in a different place (`.pi` subdir under `aidev` user's $HOME, and/or $NPM_CONFIG_PREFIX dir). diff --git a/package.json b/package.json index a00da64..3c869b4 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,10 @@ "bin": { "skynot": "bin/skynot" }, + "files": [ + "dist/", + "patches/context-lens/" + ], "scripts": { "dist": "mkdir -p bin && echo '#!/usr/bin/env node' > bin/skynot && echo \"require('../dist/index.js');\" >> bin/skynot && chmod +x bin/skynot", "build": "npm install && npx tsc && npm run dist", diff --git a/patches/context-lens/0001-Properly-calculate-costs-when-using-PPQ.ai-models.patch b/patches/context-lens/0001-Properly-calculate-costs-when-using-PPQ.ai-models.patch new file mode 100644 index 0000000..7a59edb --- /dev/null +++ b/patches/context-lens/0001-Properly-calculate-costs-when-using-PPQ.ai-models.patch @@ -0,0 +1,126 @@ +From dd5612ad3c2ff8b1fbdaa7edf8e257c3242303f5 Mon Sep 17 00:00:00 2001 +From: webwarrior-ws +Date: Tue, 2 Jun 2026 13:51:13 +0200 +Subject: [PATCH] Properly calculate costs when using PPQ.ai models + +Use cost data from PPQ.ai API [1] to get token costs for PPQ +models. This does not take caching into account because such +info is not present in [1]. + +Fixes https://github.com/larsderidder/context-lens/issues/60 + +[1] https://api.ppq.ai/v1/models +--- + src/server/store.ts | 70 ++++++++++++++++++++++++++++++++++++++++++--- + 1 file changed, 66 insertions(+), 4 deletions(-) + +diff --git a/src/server/store.ts b/src/server/store.ts +index 1afc3c5..6742964 100644 +--- a/src/server/store.ts ++++ b/src/server/store.ts +@@ -54,6 +54,50 @@ type StoreChangeListener = (event: StoreChangeEvent) => void; + const CODEX_SESSION_TTL_MS = 5 * 60 * 1000; + const GEMINI_SESSION_TTL_MS = 5 * 60 * 1000; + ++interface PpqPricing { ++ input_per_1M_tokens: number; ++ output_per_1M_tokens: number; ++} ++ ++interface PpqModelsData { ++ data: [ ++ { ++ id: string; ++ pricing: PpqPricing; ++ }, ++ ]; ++} ++ ++class PpqPricingData { ++ private static instance: PpqPricingData | null = null; ++ private data: Map = new Map(); ++ ++ private constructor(modelsData: PpqModelsData) { ++ for (const model of modelsData.data) { ++ // PPQ API uses price of -1 for automatic models. Ignore them to not have negative costs. ++ if ( ++ model.pricing.input_per_1M_tokens > 0 && ++ model.pricing.output_per_1M_tokens > 0 ++ ) { ++ this.data.set(model.id, model.pricing); ++ } ++ } ++ } ++ ++ public static async getInstance(): Promise { ++ if (!PpqPricingData.instance) { ++ const response = await fetch("https://api.ppq.ai/v1/models"); ++ const json = await response.json(); ++ PpqPricingData.instance = new PpqPricingData(json as PpqModelsData); ++ } ++ return PpqPricingData.instance; ++ } ++ ++ public getPricing(model: string): PpqPricing | undefined { ++ return this.data.get(model); ++ } ++} ++ + export class Store { + private readonly dataDir: string; + private readonly stateFile: string; +@@ -87,6 +131,9 @@ export class Store { + // Tags storage + private tagsStore: TagsStore; + ++ // PPQ pricing data. Fetched from PPQ API when Store is created. ++ private ppqPricingData: PpqPricingData | undefined; ++ + constructor(opts: { + dataDir: string; + stateFile: string; +@@ -113,6 +160,10 @@ export class Store { + } + + this.tagsStore = new TagsStore(this.dataDir); ++ ++ PpqPricingData.getInstance().then((instance) => { ++ this.ppqPricingData = instance; ++ }); + } + + getRevision(): number { +@@ -431,15 +482,26 @@ export class Store { + httpStatus === null || (httpStatus >= 200 && httpStatus < 300); + const inputTok = usage.inputTokens || contextInfo.totalTokens; + const outputTok = usage.outputTokens; +- const costUsd = isSuccessResponse +- ? estimateCost( ++ let costUsd: number | null = 0; ++ if (isSuccessResponse) { ++ if (meta?.targetUrl?.includes("ppq.ai")) { ++ const pricing = this.ppqPricingData?.getPricing(contextInfo.model); ++ if (pricing) { ++ costUsd = ++ (inputTok * pricing.input_per_1M_tokens + ++ outputTok * pricing.output_per_1M_tokens) / ++ 1000000; ++ } ++ } else { ++ costUsd = estimateCost( + contextInfo.model, + inputTok, + outputTok, + usage.cacheReadTokens, + usage.cacheWriteTokens, +- ) +- : 0; ++ ); ++ } ++ } + + const entry: CapturedEntry = { + id: this.nextEntryId++, +-- +2.54.0 + diff --git a/patches/context-lens/0002-Increase-no-traffic-warning-timeout.patch b/patches/context-lens/0002-Increase-no-traffic-warning-timeout.patch new file mode 100644 index 0000000..753d307 --- /dev/null +++ b/patches/context-lens/0002-Increase-no-traffic-warning-timeout.patch @@ -0,0 +1,26 @@ +From a1e0e854fa6a12db7cfcb32ef8a1c86fbaddab7d Mon Sep 17 00:00:00 2001 +From: Skynot Assistant +Date: Wed, 27 May 2026 14:10:00 +0200 +Subject: [PATCH] Increase no-traffic warning timeout to 45 seconds + +It might take a while for Pi to start up and make its first requests if some of +your loaded extensions gather metadata or custom model details from APIs on start. +--- + src/cli.ts | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/cli.ts b/src/cli.ts +index c479f07..d471dfc 100644 +--- a/src/cli.ts ++++ b/src/cli.ts +@@ -704,7 +704,7 @@ if (parsedArgs.commandName === "analyze") { + ); + req.on("error", () => {}); + req.on("timeout", () => req.destroy()); +- }, 15_000); ++ }, 45_000); + } + } + +-- +2.54.0 diff --git a/src/index.ts b/src/index.ts index 19d4ed6..67e4a15 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,6 +28,7 @@ const AGENT_NPM_PACKAGE = "@earendil-works/pi-coding-agent"; const AGENT_GITHUB_REPO = "earendil-works/pi"; const AGENT_USER = "aidev"; const LAUNCHER_SCRIPT_FILENAME = "spi"; +const CONTEXT_LENS_SCRIPT_FILENAME = "cpi"; const AGENT_GROUP_NAME = "aiteam"; const DEFAULT_UMASK = "007"; const MIN_NODE_MAJOR_VERSION = 22; @@ -81,6 +82,16 @@ type RunProcessOptions = { verboseStdErr?: boolean; }; +function getProcessOptions(verbose?: boolean, cwd?: string) { + const opts: RunProcessOptions = Empty.object(); + if (verbose) { + opts.verboseStdErr = true; + opts.verboseStdOut = true; + } + opts.cwd = cwd; + return opts; +} + function runCommand( command: string, args: string[], @@ -615,12 +626,13 @@ set_dir_umask } async function createLauncherScript( - piBinaryPath: string, + command: string, + scriptFileName: string, apiKeyExport: Option<{ name: string; value: string }> = Nothing ): Promise { const currentUserHome = os.homedir(); const binDir = path.join(currentUserHome, "bin"); - const scriptPath = path.join(binDir, LAUNCHER_SCRIPT_FILENAME); + const scriptPath = path.join(binDir, scriptFileName); console.log(`Creating launcher script at ${scriptPath}...`); @@ -704,9 +716,7 @@ if [ \${#EXPOSED_DIRS[@]} -gt 0 ]; then echo "" fi -FULL_SUDO_CMD="${exportPrefix}export npm_config_prefix=$AGENT_USER_HOME/.npm-global && umask ${DEFAULT_UMASK} && cd $CURRENT_DIR && ${piBinaryPath} $@" -echo "Launching Pi with ${AGENT_USER} user (sudo is required to impersonate '${AGENT_USER}' user)..." -exec sudo -i -u ${AGENT_USER} bash -c "$FULL_SUDO_CMD" +${exportPrefix} ${command} `; fs.writeFileSync(scriptPath, scriptContent, { mode: 0o755 }); console.log("Launcher script created."); @@ -730,6 +740,33 @@ exec sudo -i -u ${AGENT_USER} bash -c "$FULL_SUDO_CMD" } } +async function createPiLauncherScript( + piBinaryPath: string, + apiKeyExport: Option<{ name: string; value: string }> = Nothing +): Promise { + const command = ` +FULL_SUDO_CMD="export npm_config_prefix=$AGENT_USER_HOME/.npm-global && umask ${DEFAULT_UMASK} && cd $CURRENT_DIR && ${piBinaryPath} $@" +echo "Launching Pi with ${AGENT_USER} user (sudo is required to impersonate '${AGENT_USER}' user)..." +exec sudo -i -u ${AGENT_USER} bash -c "$FULL_SUDO_CMD"`; + await createLauncherScript(command, LAUNCHER_SCRIPT_FILENAME, apiKeyExport); +} + +async function createContextLensLauncherScript( + contextLensDir: string, + apiKeyExport: Option<{ name: string; value: string }> = Nothing +): Promise { + const cmd = `HOME=${agentUserHome} UPSTREAM_OPENAI_URL=https://api.ppq.ai node ${contextLensDir}/dist/cli.js pi`; + const command = ` +FULL_SUDO_CMD="export npm_config_prefix=$AGENT_USER_HOME/.npm-global && umask ${DEFAULT_UMASK} && cd $CURRENT_DIR && ${cmd} $@" +echo "Launching Pi using context-lens warapper with ${AGENT_USER} user (sudo is required to impersonate '${AGENT_USER}' user)..." +exec sudo -i -u ${AGENT_USER} bash -c "$FULL_SUDO_CMD"`; + await createLauncherScript( + command, + CONTEXT_LENS_SCRIPT_FILENAME, + apiKeyExport + ); +} + async function createMacOsGroup( sudoReason: string, freeGroupIdFindingCount: number @@ -933,7 +970,7 @@ async function setupWorkDir(): Promise { } const RECOMMENDED_EXTENSIONS = [ - "npm:awto-pi-lot", + "git:github.com/webwarrior-ws/awto-pi-lot", // BEWARE: this extension doesn't have NPM Provenance enabled yet: "npm:pi-wtf", @@ -950,6 +987,118 @@ async function installExtensions( } } +async function buildContextLens( + contextLensDir: string, + verbose?: boolean +): Promise { + console.log("Building context-lens..."); + const commandOptions = getProcessOptions(verbose); + + let usePnpm = false; + try { + await execAsync("which pnpm"); + usePnpm = true; + } catch { + // pnpm is not available + } + + if (usePnpm) { + for (const dir of [contextLensDir, path.join(contextLensDir, "ui")]) { + commandOptions.cwd = dir; + await runCommand("pnpm", ["install"], commandOptions); + await runCommand("pnpm", ["run", "build"], commandOptions); + } + } else { + commandOptions.cwd = contextLensDir; + await runCommand("npm", ["install"], commandOptions); + await runCommand("npm", ["run", "generate:version"], commandOptions); + await runCommand("npm", ["run", "generate:types"], commandOptions); + await runCommand("npx", ["tsc"], commandOptions); + + commandOptions.cwd = path.join(contextLensDir, "ui"); + await runCommand("npm", ["install"], commandOptions); + await runCommand("npm", ["run", "build"], commandOptions); + } + + console.log("context-lens built."); +} + +async function installContextLens( + update: boolean, + apiKeyExport: Option<{ name: string; value: string }> = Nothing, + verbose?: boolean +): Promise { + const contextLensRepoName = "context-lens"; + const contextLensGithubRepoUrl = `https://github.com/larsderidder/${contextLensRepoName}.git`; + const contextLensDir = path.join(agentUserHome, contextLensRepoName); + const commandOptions = getProcessOptions(verbose, agentUserHome); + const commandOptionsForContextLensDir = getProcessOptions( + verbose, + contextLensDir + ); + + async function applyPatches() { + const patchesDir = path.join( + __dirname, + "..", + "patches", + "context-lens" + ); + const patchFiles = fs + .readdirSync(patchesDir) + .filter((fileName) => fileName.endsWith(".patch")); + + for (const patchFile of patchFiles) { + const patchPath = path.join(patchesDir, patchFile); + console.log(`Applying patch: ${patchFile}`); + await runCommand( + "git", + ["apply", patchPath], + commandOptionsForContextLensDir + ); + } + } + + if (fs.existsSync(contextLensDir)) { + console.log("context-lens already installed."); + const cliPath = path.join(contextLensDir, "dist", "cli.js"); + const uiDistPath = path.join(contextLensDir, "ui", "dist"); + if (update || !fs.existsSync(cliPath) || !fs.existsSync(uiDistPath)) { + console.log( + "Updating or building missing context-lens distribution..." + ); + if (update) { + await runCommand( + "git", + ["fetch"], + commandOptionsForContextLensDir + ); + await runCommand( + "git", + ["reset", "--hard", "origin/main"], + commandOptionsForContextLensDir + ); + await applyPatches(); + } + await buildContextLens(contextLensDir, verbose); + console.log("context-lens built successfully."); + } + } else { + console.log("Installing context-lens..."); + await runAsAgentUser(`git clone ${contextLensGithubRepoUrl}`, verbose); + // mark context-lens dir as safe for git + const safeDirectoryCmd = `git config --global --add safe.directory '${contextLensDir}'`; + await execAsync(safeDirectoryCmd); + + await applyPatches(); + + await buildContextLens(contextLensDir, verbose); + console.log("context-lens installed."); + } + + await createContextLensLauncherScript(contextLensDir, apiKeyExport); +} + async function launchAgent(): Promise { const scriptPath = path.join(os.homedir(), "bin", LAUNCHER_SCRIPT_FILENAME); const child = spawn(scriptPath, [], { stdio: "inherit" }); @@ -1185,6 +1334,9 @@ async function destroyInstallation(): Promise { ); console.log(` - The '${AGENT_GROUP_NAME}' group`); console.log(` - The launcher script ~/bin/${LAUNCHER_SCRIPT_FILENAME}`); + console.log( + ` - The launcher script ~/bin/${CONTEXT_LENS_SCRIPT_FILENAME}` + ); console.log(""); const confirmation = await askQuestion( @@ -1242,16 +1394,17 @@ async function destroyInstallation(): Promise { ); } - // Remove the launcher script - const launcherPath = path.join( - os.homedir(), - "bin", - LAUNCHER_SCRIPT_FILENAME - ); - if (fs.existsSync(launcherPath)) { - console.log(`Removing launcher script at ${launcherPath}...`); - fs.unlinkSync(launcherPath); - console.log("Launcher script removed."); + // Remove the launcher scripts + for (const scriptFileName of [ + LAUNCHER_SCRIPT_FILENAME, + CONTEXT_LENS_SCRIPT_FILENAME, + ]) { + const launcherPath = path.join(os.homedir(), "bin", scriptFileName); + if (fs.existsSync(launcherPath)) { + console.log(`Removing launcher script at ${launcherPath}...`); + fs.unlinkSync(launcherPath); + console.log("Launcher script removed."); + } } console.log("\n=== DESTROY COMPLETE ==="); @@ -1276,6 +1429,10 @@ async function main() { "-e, --extensions", `DEPRECATED: rather use \`${LAUNCHER_SCRIPT_FILENAME} install \` instead, after install.` ) + .option( + "-c, --context-lens", + `This flag additionaly installs context-lens after installing Pi and creates a launcher script "cpi" for it.` + ) .option( "-a, --auth", `Prompt for AI model's credentials to add env var to launcher script, or to create an auth.json file.` @@ -1447,7 +1604,11 @@ async function main() { apiKeyExport = await configureAuth(); } - await createLauncherScript(piBinaryPath, apiKeyExport); + await createPiLauncherScript(piBinaryPath, apiKeyExport); + + if (opts.contextLens) { + await installContextLens(opts.update, apiKeyExport, opts.verbose); + } const workDir = await setupWorkDir(); console.log( @@ -1461,6 +1622,11 @@ async function main() { console.log(`3. Clone the git repository where you will work on`); console.log(`4. \`cd\` into the cloned repository`); console.log(`5. Launch via \`${LAUNCHER_SCRIPT_FILENAME}\`\n`); + if (opts.contextLens) { + console.log( + `6. Launch with context-lens via \`${CONTEXT_LENS_SCRIPT_FILENAME}\`\n` + ); + } } main().catch((err) => {