Exercise polyglot candidates with the actual MCP probe

Measure protocol errors separately from stderr diagnostics and test
Windows without Git or Unix shells on PATH. Compare Node and shell
polyglots on Node 22 and 24 across all three platforms.

Author: jonathanhefner

.github/workflows/polyglot-prototype.yml

@@ -11,19 +11,20 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
+        node: [22, 24]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v7.0.1
       - uses: pnpm/action-setup@v6.1.0
       - uses: actions/setup-node@v7.0.0
         with:
-          node-version: 24
+          node-version: ${{ matrix.node }}
           cache: pnpm
       - run: pnpm install --frozen-lockfile
       - name: Probe launcher routes
         shell: bash
-        run: node scripts/polyglot-probe.mjs > prototype-result.json
+        run: node scripts/polyglot-mcp-probe.mjs > prototype-result.json
       - uses: actions/upload-artifact@v4
         with:
-          name: result-${{ matrix.os }}
+          name: result-${{ matrix.os }}-node${{ matrix.node }}
           path: prototype-result.json

scripts/polyglot-mcp-probe.mjs

@@ -0,0 +1,55 @@
+import assert from 'node:assert/strict';
+import {mkdtemp, mkdir, writeFile, cp, chmod, rm, realpath} from 'node:fs/promises';
+import {tmpdir} from 'node:os';
+import {dirname,join,delimiter} from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {Client} from '@modelcontextprotocol/sdk/client/index.js';
+import {StdioClientTransport} from '@modelcontextprotocol/sdk/client/stdio.js';
+const repo=fileURLToPath(new URL('..',import.meta.url));
+const parent=await mkdtemp(join(tmpdir(),'apc polyglot MCP '));
+const root=join(parent,'plugin with spaces');
+await cp(join(repo,'plugins/agent-plugins-conformance-core'),root,{recursive:true});
+await mkdir(join(root,'bin'));
+await writeFile(join(root,'bin/package.json'),'{"type":"commonjs"}\n');
+const jsBody='0</* :\n@node "%~dp0..\\dist\\probe.mjs" %*\n@exit /b %errorlevel%\n*/0;\nimport("../dist/probe.mjs");\n';
+const shBody=':; exec node "$(dirname "$0")/../dist/probe.mjs" "$@"\n@echo off\nnode "%~dp0..\\dist\\probe.mjs" %*\nexit /b %errorlevel%\n';
+const variants={
+ node_plain:'#!/usr/bin/env node\n'+jsBody,
+ node_envS:'#!/usr/bin/env -S node # 2>nul\n'+jsBody,
+ node_envS_double_space:'#!/usr/bin/env  -S node # 2>nul\n'+jsBody,
+ shell_envS:'#!/usr/bin/env -S sh # 2>nul\n'+shBody,
+};
+const env={...process.env,PLUGIN_ROOT:root,PLUGIN_DATA:parent};
+if(process.platform==='win32') {
+ for(const key of Object.keys(env)) if(key.toLowerCase()==='path') delete env[key];
+ env.PATH=[dirname(process.execPath),join(process.env.SystemRoot,'System32'),process.env.SystemRoot].join(delimiter);
+}
+const results=[];
+try {
+ for(const [name,body] of Object.entries(variants)) {
+  const file=join(root,'bin',name+'.cmd'); await writeFile(file,body);await chmod(file,0o755);
+  const routes={sdk:{command:file,args:['relative']}};
+  if(process.platform==='win32') for(const quiet of [false,true]) routes[quiet?'cmdQuiet':'cmd']={command:process.env.ComSpec||'cmd.exe',args:['/d',...(quiet?['/q']:[]),'/s','/c',`"${file}" relative`]};
+  for(const [route,launch] of Object.entries(routes)){
+   const client=new Client({name:'polyglot-prototype',version:'1'});
+   const transport=new StdioClientTransport({...launch,cwd:join(root,'probe-workdir'),env,stderr:'pipe'});
+   const row={name,route,launch,stderr:'',protocolErrors:[],success:false};
+   client.onerror=error=>row.protocolErrors.push(String(error));
+   transport.stderr?.on('data',data=>row.stderr+=data.toString());
+   try{
+    await client.connect(transport,{timeout:10000});
+    const result=await client.callTool({name:'observe',arguments:{}},undefined,{timeout:10000});
+    assert.ok(!result.isError);
+    const observation=result.structuredContent;
+    assert.equal(observation.evidence.root,await realpath(root));
+    assert.equal(observation.evidence.cwd,await realpath(join(root,'probe-workdir')));
+    assert.deepEqual(observation.evidence.argv,['relative']);
+    row.observation=observation; row.success=true;
+   }catch(error){row.error=String(error)}
+   finally{await client.close();}
+   row.protocolClean=row.success&&row.protocolErrors.length===0;
+   results.push(row);
+  }
+ }
+}finally{await rm(parent,{recursive:true,force:true});}
+console.log(JSON.stringify({platform:process.platform,node:process.version,path:env.PATH,results},null,2));