diff --git a/.github/workflows/release-python-snapshots.yml b/.github/workflows/release-python-snapshots.yml index 18a7240de2d..3f27291c222 100644 --- a/.github/workflows/release-python-snapshots.yml +++ b/.github/workflows/release-python-snapshots.yml @@ -23,7 +23,7 @@ jobs: secrets: BAZEL_CACHE_KEY: ${{ secrets.BAZEL_CACHE_KEY }} WORKERS_MIRROR_URL: ${{ secrets.WORKERS_MIRROR_URL }} - GOOGLESOURCE_COOKE: ${{ secrets.GOOGLESOURCE_COOKIE }} + GOOGLESOURCE_COOKIE: ${{ secrets.GOOGLESOURCE_COOKIE }} build: needs: [build-linux] diff --git a/build/deps/python.MODULE.bazel b/build/deps/python.MODULE.bazel index 74f03c39d48..c2e01923506 100644 --- a/build/deps/python.MODULE.bazel +++ b/build/deps/python.MODULE.bazel @@ -18,4 +18,4 @@ pip.parse( use_repo(pip, "py_deps", "v8_python_deps") pyodide = use_extension("//build/deps:dep_pyodide.bzl", "pyodide") -use_repo(pyodide, "all_pyodide_wheels_20240829.4", "all_pyodide_wheels_20250808", "beautifulsoup4_src_0.26.0a2", "beautifulsoup4_src_0.28.2", "beautifulsoup4_src_development", "fastapi_src_0.26.0a2", "fastapi_src_0.28.2", "fastapi_src_development", "pyodide-0.26.0a2", "pyodide-0.28.2", "pyodide-lock_20240829.4.json", "pyodide-lock_20250808.json", "pyodide-snapshot-baseline-4569679fb.bin", "pyodide-snapshot-baseline-61eedf943.bin", "pyodide-snapshot-ew-py-package-snapshot_fastapi-v2.bin", "pyodide-snapshot-ew-py-package-snapshot_numpy-v2.bin", "pyodide-snapshot-package_snapshot_fastapi-a6ccb56fe.bin", "pyodide-snapshot-package_snapshot_numpy-60c9cb28e.bin", "pyodide-snapshot-snapshot_a6b652a95810783f5078b9a5dbd4a07c30718acb4ff724e82c25db7353dd7f2d.bin", "pyodide_0.26.0a2_2024-03-01_79.capnp.bin", "pyodide_0.28.2_2025-01-16_10.capnp.bin", "pyodide_dev.capnp.bin", "pytest-asyncio_src_0.26.0a2", "pytest-asyncio_src_0.28.2", "pytest-asyncio_src_development", "python-workers-runtime-sdk_src_0.26.0a2", "python-workers-runtime-sdk_src_0.28.2", "python-workers-runtime-sdk_src_development", "scipy_src_0.26.0a2", "shapely_src_0.28.2", "shapely_src_development") +use_repo(pyodide, "all_pyodide_wheels_20240829.4", "all_pyodide_wheels_20250808", "beautifulsoup4_src_0.26.0a2", "beautifulsoup4_src_0.28.2", "beautifulsoup4_src_314.0.0a1", "beautifulsoup4_src_development", "fastapi_src_0.26.0a2", "fastapi_src_0.28.2", "pyodide-0.26.0a2", "pyodide-0.28.2", "pyodide-314.0.0a1", "pyodide-lock_20240829.4.json", "pyodide-lock_20250808.json", "pyodide-snapshot-baseline-4569679fb.bin", "pyodide-snapshot-baseline-52636df0a.bin", "pyodide-snapshot-baseline-61eedf943.bin", "pyodide-snapshot-ew-py-package-snapshot_fastapi-v2.bin", "pyodide-snapshot-ew-py-package-snapshot_numpy-v2.bin", "pyodide-snapshot-package_snapshot_fastapi-a6ccb56fe.bin", "pyodide-snapshot-package_snapshot_numpy-60c9cb28e.bin", "pyodide-snapshot-snapshot_a6b652a95810783f5078b9a5dbd4a07c30718acb4ff724e82c25db7353dd7f2d.bin", "pyodide_0.26.0a2_2024-03-01_79.capnp.bin", "pyodide_0.28.2_2025-01-16_10.capnp.bin", "pyodide_314.0.0a1_2026-04-09_4.capnp.bin", "pyodide_dev.capnp.bin", "pytest-asyncio_src_0.26.0a2", "pytest-asyncio_src_0.28.2", "pytest-asyncio_src_314.0.0a1", "pytest-asyncio_src_development", "python-workers-runtime-sdk_src_0.26.0a2", "python-workers-runtime-sdk_src_0.28.2", "python-workers-runtime-sdk_src_314.0.0a1", "python-workers-runtime-sdk_src_development", "scipy_src_0.26.0a2", "shapely_src_0.28.2") diff --git a/build/python_metadata.bzl b/build/python_metadata.bzl index 0cbfacd9cb9..66cfff02d78 100644 --- a/build/python_metadata.bzl +++ b/build/python_metadata.bzl @@ -22,6 +22,10 @@ PYODIDE_VERSIONS = [ "version": "0.28.2", "sha256": "c9f6dd067d119e50850849f7428e3c636ecbc2684a0d2ff992f3bd48a1062b6c", }, + { + "version": "314.0.0a1", + "sha256": "7cceacea2efb493f30667192f0380e1fc6955c38fe34897b2d5be98023758073", + }, ] # This is the list of all the package metadata that we use. @@ -34,6 +38,8 @@ PYODIDE_VERSIONS = [ _package_lockfiles = [ PACKAGES_20240829_4, PACKAGES_20250808, + # As of Pyodide 314, we don't bundle any packages by default + # So this list does not need to be updated anymore. ] # The below is a list of pyodide-lock.json files for each package bundle version that we support. @@ -69,6 +75,7 @@ def _add_integrity(entry): def _make_vendored_packages(entry): if entry["name"] == "development": return + vendor_tests = {} for e in entry["vendored_packages_for_tests"]: vendor_tests[e["name"]] = e @@ -158,6 +165,7 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([ }, { "name": "0.28.2", + "released": True, "pyodide_version": "0.28.2", "pyodide_date": "2025-01-16", "packages": PACKAGES_20250808, @@ -181,11 +189,6 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([ "abi": "3.13", "sha256": "955091f1bd2eb33255ff2633df990bedc96e2f6294e78f2b416078777394f942", }, - # { - # "name": "scipy", - # "abi": "3.13", - # "sha256": "4f1b6fc179bd5c6d3de68abc4aa9fca2aaecd09c5c8d357c2ecfedce7d621f3d", - # }, { "name": "shapely", "abi": "3.13", @@ -194,7 +197,21 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([ ], }, { - "real_pyodide_version": "0.28.2", + "name": "314.0.0a1", + "pyodide_version": "314.0.0a1", + "pyodide_date": "2026-04-09", + "backport": "4", + "integrity": "sha256-T8JMW0IoonHdZTqlvHgkQrHT6ndyXXfnbbUP3UO15ag=", + "flag": "pythonWorkersDevPyodide", + "enable_flag_name": "python_workers_development", + "emscripten_version": "5.0.3", + "python_version": "3.14.2", + "baseline_snapshot": "baseline-52636df0a.bin", + "baseline_snapshot_hash": "52636df0a566fe69bc762796c458603bc69aad434b59e16e1d46c9bc8a2f04bf", + "vendored_packages_for_tests": VENDORED_VERSION_INDEPENDENT, + }, + { + "real_pyodide_version": "314.0.0a1", "name": "development", "pyodide_version": "dev", "pyodide_date": "dev", diff --git a/src/pyodide/helpers.bzl b/src/pyodide/helpers.bzl index e2f3d97ff12..dd20a40fa60 100644 --- a/src/pyodide/helpers.bzl +++ b/src/pyodide/helpers.bzl @@ -42,11 +42,11 @@ def _copy_and_capnp_embed(src): def _fmt_python_snapshot_release( pyodide_version, pyodide_date, - packages, backport, baseline_snapshot_hash, flag, real_pyodide_version, + packages = "", **_kwds): content = ", ".join( [ @@ -162,20 +162,9 @@ import { } from "pyodide-internal:pool/builtin_wrappers"; """ -# pyodide.asm.js patches +# pyodide.asm.mjs patches # TODO: all of these should be fixed by linking our own Pyodide or by upstreaming. -_REPLACEMENTS = [ - [ - # Convert pyodide.asm.js into an es6 module. - # When we link our own we can pass `-sES6_MODULE` to the linker and it will do this for us - # automatically. - "var _createPyodideModule", - _PRELUDE + "export const _createPyodideModule", - ], - [ - "globalThis._createPyodideModule = _createPyodideModule;", - "", - ], +_REPLACEMENTS_COMMON = [ [ "new WebAssembly.Module", "newWasmModule", @@ -227,15 +216,6 @@ _REPLACEMENTS = [ "getMemory(", "Module.getMemoryPatched(Module, libName, ", ], - # to fix RPC, applies https://github.com/pyodide/pyodide/commit/8da1f38f7 - [ - "nullToUndefined(func.apply(", - "nullToUndefined(patchedApplyFunc(API, func, ", - ], - [ - "nullToUndefined(Function.prototype.apply.apply", - "nullToUndefined(API.config.jsglobals.Function.prototype.apply.apply", - ], [ "function _PyEM_CountFuncParams(func){", "function _PyEM_CountFuncParams(func){ return patched_PyEM_CountFuncParams(Module, func);", @@ -253,13 +233,74 @@ _REPLACEMENTS = [ ], ] -def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_js = None, python_stdlib_zip = None, emscripten_setup_override = None): +_REPLACEMENTS = { + "0.26.0a2": _REPLACEMENTS_COMMON + [ + # for 0.28.2 or earlier, pyodide.asm.js was a commonjs module + [ + # Convert pyodide.asm.js into an es6 module. + # When we link our own we can pass `-sES6_MODULE` to the linker and it will do this for us + # automatically. + "var _createPyodideModule", + _PRELUDE + "export const _createPyodideModule", + ], + [ + "globalThis._createPyodideModule = _createPyodideModule;", + "", + ], + # to fix RPC, applies https://github.com/pyodide/pyodide/commit/8da1f38f7 + [ + "nullToUndefined(func.apply(", + "nullToUndefined(patchedApplyFunc(API, func, ", + ], + [ + "nullToUndefined(Function.prototype.apply.apply", + "nullToUndefined(API.config.jsglobals.Function.prototype.apply.apply", + ], + ], + "0.28.2": _REPLACEMENTS_COMMON + [ + # for 0.28.2 or earlier, pyodide.asm.js was a commonjs module + [ + # Convert pyodide.asm.js into an es6 module. + # When we link our own we can pass `-sES6_MODULE` to the linker and it will do this for us + # automatically. + "var _createPyodideModule", + _PRELUDE + "export const _createPyodideModule", + ], + [ + "globalThis._createPyodideModule = _createPyodideModule;", + "", + ], + # to fix RPC, applies https://github.com/pyodide/pyodide/commit/8da1f38f7 + [ + "nullToUndefined(func.apply(", + "nullToUndefined(patchedApplyFunc(API, func, ", + ], + [ + "nullToUndefined(Function.prototype.apply.apply", + "nullToUndefined(API.config.jsglobals.Function.prototype.apply.apply", + ], + ], + "314.0.0a1": _REPLACEMENTS_COMMON + [ + # for 314 or later, pyodide.asm.mjs is es6 module + [ + "export default _createPyodideModule;", + # still expose _createPyodideModule for compatibility (import { _createPyodideModule }) + _PRELUDE + "export default _createPyodideModule; export { _createPyodideModule };", + ], + ], +} + +def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_mjs = None, python_stdlib_zip = None, emscripten_setup_override = None): pyodide_package = "@pyodide-%s//" % version if not pyodide_asm_wasm: pyodide_asm_wasm = pyodide_package + ":pyodide/pyodide.asm.wasm" - if not pyodide_asm_js: - pyodide_asm_js = pyodide_package + ":pyodide/pyodide.asm.js" + if not pyodide_asm_mjs: + if version in ("0.26.0a2", "0.28.2"): + pyodide_asm_mjs = pyodide_package + ":pyodide/pyodide.asm.js" + else: + # for 314 or later, pyodide.asm.mjs is es6 module + pyodide_asm_mjs = pyodide_package + ":pyodide/pyodide.asm.mjs" if not python_stdlib_zip: python_stdlib_zip = pyodide_package + ":pyodide/python_stdlib.zip" @@ -269,21 +310,30 @@ def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_js = None, p _copy_to_generated(python_stdlib_zip, version, out_name = "python_stdlib.zip") expand_template( - name = "pyodide.asm.js@rule@" + version, - out = _out_path("pyodide.asm.js", version), - substitutions = dict(_REPLACEMENTS), - template = pyodide_asm_js, + name = "pyodide.asm.mjs@rule@" + version, + out = _out_path("pyodide.asm.mjs", version), + substitutions = dict(_REPLACEMENTS[version]), + template = pyodide_asm_mjs, ) js_file( - name = "pyodide.asm.js@rule_js@" + version, - srcs = [_out_path("pyodide.asm.js", version)], - deps = ["pyodide.asm.js@rule@" + version], + name = "pyodide.asm.mjs@rule_js@" + version, + srcs = [_out_path("pyodide.asm.mjs", version)], + deps = ["pyodide.asm.mjs@rule@" + version], ) if emscripten_setup_override: _copy_to_generated(out_name = "emscriptenSetup.js", name = "emscriptenSetup", src = emscripten_setup_override, version = version) else: + expand_template( + name = "esbuild.config.mjs@" + version, + out = _out_path("esbuild.config.mjs", version), + substitutions = { + "%PYODIDE_VERSION%": version, + }, + template = "internal/pool/esbuild.config.mjs", + ) + esbuild( name = "emscriptenSetup@" + version, # exclude emscriptenSetup from source set so that rules_ts won't also try to create a JS output @@ -291,11 +341,11 @@ def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_js = None, p srcs = native.glob([ "internal/pool/*.ts", ], exclude = ["internal/pool/emscriptenSetup.ts"]) + [ - _out_path("pyodide.asm.js", version), + _out_path("pyodide.asm.mjs", version), "internal/util.ts", "internal/const.ts", ], - config = "internal/pool/esbuild.config.mjs", + config = _out_path("esbuild.config.mjs", version), entry_point = "internal/pool/emscriptenSetup.ts", external = [ "child_process", @@ -312,11 +362,15 @@ def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_js = None, p "node:url", "node:vm", "internal:unsafe-eval", + "node:stream", + "node:tls", + "node:net", + "node:module", ], format = "esm", output = _out_path("emscriptenSetup.js", version), target = "esnext", - deps = ["pyodide.asm.js@rule_js@" + version], + deps = ["pyodide.asm.mjs@rule_js@" + version], ) import_name = "pyodideRuntime" diff --git a/src/pyodide/internal/const.ts b/src/pyodide/internal/const.ts index c971a313b14..118c66cd245 100644 --- a/src/pyodide/internal/const.ts +++ b/src/pyodide/internal/const.ts @@ -5,4 +5,5 @@ export const PyodideVersion = { V0_26_0a2: '0.26.0a2', V0_28_2: '0.28.2', + V314_0_0a1: '314.0.0a1', } as const; diff --git a/src/pyodide/internal/pool/emscriptenSetup.ts b/src/pyodide/internal/pool/emscriptenSetup.ts index 769d67d1dbf..4bd9b170d38 100644 --- a/src/pyodide/internal/pool/emscriptenSetup.ts +++ b/src/pyodide/internal/pool/emscriptenSetup.ts @@ -17,7 +17,7 @@ import { finishSetup, } from 'pyodide-internal:pool/builtin_wrappers'; -import { getSentinelImport } from 'pyodide-internal:pool/sentinel'; +import { getJsvErrorImport } from 'pyodide-internal:pool/sentinel'; /** * A preRun hook. Make sure environment variables are visible at runtime. @@ -110,7 +110,7 @@ function getPrepareFileSystem(pythonStdlib: ArrayBuffer): PreRunHook { function getInstantiateWasm( pyodideWasmModule: WebAssembly.Module ): EmscriptenSettings['instantiateWasm'] { - const sentinelImportPromise = getSentinelImport(); + const jsvErrorImportPromise = getJsvErrorImport(); return function instantiateWasm( wasmImports: WebAssembly.Imports, successCallback: ( @@ -119,8 +119,20 @@ function getInstantiateWasm( ) => void ): WebAssembly.Exports { (async function (): Promise { - wasmImports.sentinel = await sentinelImportPromise; - // Instantiate pyodideWasmModule with wasmImports + const { Jsv_GetError_import, JsvError_Check } = + await jsvErrorImportPromise; + + // Pyodide <= 0.28.2: These were named create_sentinel and is_sentinel + // Pyodide >= 314: These are renamed to Jsv_GetError_import and JsvError_Check and moved to env namespace + wasmImports.sentinel = { + create_sentinel: Jsv_GetError_import, + is_sentinel: JsvError_Check, + }; + const env = wasmImports.env; + if (env) { + env.Jsv_GetError_import = Jsv_GetError_import; + env.JsvError_Check = JsvError_Check; + } const instance = await WebAssembly.instantiate( pyodideWasmModule, wasmImports @@ -168,7 +180,15 @@ function getEmscriptenSettings( (res) => (config.resolveLockFilePromise = res) ); } - const API = { config, lockFilePromise }; + const API = { + config, + lockFilePromise, + runtimeEnv: { + IN_WORKERD: true, + IN_BROWSER: true, + IN_BROWSER_MAIN_THREAD: true, + }, + }; let resolveReadyPromise: (mod: Module) => void; let rejectReadyPromise: (e: any) => void = () => {}; const readyPromise: Promise = new Promise((res, rej) => { diff --git a/src/pyodide/internal/pool/esbuild.config.mjs b/src/pyodide/internal/pool/esbuild.config.mjs index 17eab11f841..a3bee10453c 100644 --- a/src/pyodide/internal/pool/esbuild.config.mjs +++ b/src/pyodide/internal/pool/esbuild.config.mjs @@ -6,6 +6,8 @@ const pyodideRootDir = dirname( dirname(dirname(fileURLToPath(import.meta.url))) ); +const pyodideVersion = '%PYODIDE_VERSION%'; + let resolvePlugin = { name: 'pyodide-internal', setup(build) { @@ -14,15 +16,13 @@ let resolvePlugin = { let rest = args.path.split(':')[1]; let path; if (rest.startsWith('generated')) { - // I couldn't figure out how to pass down the version, so instead we'll look through the - // directories in `pyodideRootDir` and find one that starts with a 0. This will work until - // Pyodide has a 1.0 release. - const dir = readdirSync(pyodideRootDir).filter((x) => - x.startsWith('0') - )[0]; - path = join(pyodideRootDir, dir, rest); + path = join(pyodideRootDir, pyodideVersion, rest); if (!existsSync(path)) { - path += '.js'; + if (existsSync(path + '.js')) { + path += '.js'; + } else { + path += '.mjs'; + } } } else { path = join(pyodideRootDir, 'internal', rest); diff --git a/src/pyodide/internal/pool/sentinel.ts b/src/pyodide/internal/pool/sentinel.ts index ea59da06776..21829323ba7 100644 --- a/src/pyodide/internal/pool/sentinel.ts +++ b/src/pyodide/internal/pool/sentinel.ts @@ -1,5 +1,5 @@ -// This is https://github.com/pyodide/pyodide/blob/main/src/core/sentinel.ts. -// It goes into `pyodide.js` not `pyodide.asm.js`. We don't use `pyodide.js` so we have to reproduce +// This is https://github.com/pyodide/pyodide/blob/main/src/core/jsverror.ts. +// It goes into `pyodide.js` not `pyodide.asm.mjs`. We don't use `pyodide.js` so we have to reproduce // it here. import { wasmInstantiate } from 'pyodide-internal:pool/builtin_wrappers'; @@ -39,14 +39,31 @@ function decodeBase64(input: string): Uint8Array { return output; } -// This string is https://github.com/pyodide/pyodide/blob/main/src/core/sentinel.wat assembled and -// hex encoded. -const sentinelWasm = decodeBase64( - 'AGFzbQEAAAABDANfAGAAAW9gAW8BfwMDAgECByECD2NyZWF0ZV9zZW50aW5lbAAAC2lzX3NlbnRpbmVsAAEKEwIHAPsBAPsbCwkAIAD7GvsUAAs' +// See https://github.com/pyodide/pyodide/pull/6107 +const jsvErrorWasm = decodeBase64( + 'AGFzbQEAAAABDANfAGAAAW9gAW8BfwMDAgECBygCE0pzdl9HZXRFcnJvcl9pbXBvcnQAAA5Kc3ZFcnJvcl9DaGVjawABChMCBwD7AQD7GwsJACAA+xr7FAAL' ); +<<<<<<< HEAD export async function getSentinelImport() { const imports = {}; const { instance } = await wasmInstantiate(sentinelWasm, imports); return instance.exports; +||||||| parent of 90a8e036a (Set Pyodide 314.0.0a1 as a development version) +export async function getSentinelImport() { + const module: WebAssembly.Module = new WebAssembly.Module(sentinelWasm); + const instance = await WebAssembly.instantiate(module); + return instance.exports; +======= +export async function getJsvErrorImport(): Promise<{ + Jsv_GetError_import: () => unknown; + JsvError_Check: (val: unknown) => number; +}> { + const module: WebAssembly.Module = new WebAssembly.Module(jsvErrorWasm); + const instance = await WebAssembly.instantiate(module); + return instance.exports as { + Jsv_GetError_import: () => unknown; + JsvError_Check: (val: unknown) => number; + }; +>>>>>>> 90a8e036a (Set Pyodide 314.0.0a1 as a development version) } diff --git a/src/pyodide/internal/python.ts b/src/pyodide/internal/python.ts index 31d769f9035..bd010085b27 100644 --- a/src/pyodide/internal/python.ts +++ b/src/pyodide/internal/python.ts @@ -182,7 +182,11 @@ function getSignalClockAddr(Module: Module): number { function setupRuntimeSignalHandling(Module: Module): void { Module.Py_EmscriptenSignalBuffer = new Uint8Array(1); const version = Module.API.version; - if (version === PyodideVersion.V0_26_0a2) { + if ( + version === PyodideVersion.V0_26_0a2 || + /* TODO(before 314 release): Handle this in upstream Pyodide */ + version === PyodideVersion.V314_0_0a1 + ) { return; } if (version === PyodideVersion.V0_28_2) { diff --git a/src/pyodide/make_snapshots.py b/src/pyodide/make_snapshots.py index 5df871d9f00..f1c6e2d18f8 100644 --- a/src/pyodide/make_snapshots.py +++ b/src/pyodide/make_snapshots.py @@ -55,7 +55,6 @@ def bundle_version_info(): const mainWorker :Workerd.Worker = ( modules = [ (name = "worker.py", pythonModule = embed "./worker.py"), - {requirements} ], compatibilityDate = "2025-08-05", compatibilityFlags = ["python_no_global_handlers", {compat_flags}], @@ -67,47 +66,32 @@ def bundle_version_info(): def make_config( flags: list[str], - reqs: list[str], ) -> str: - requirements = "" - for name in reqs: - requirements += f'(name="{name}", pythonRequirement=""),' - compat_flags = "" for flag in flags: compat_flags += f'"{flag}", ' - return TEMPLATE.format(requirements=requirements, compat_flags=compat_flags) + return TEMPLATE.format(compat_flags=compat_flags) -def make_worker(imports: list[str]) -> str: - contents = "" - for i in imports: - contents += f"import {i}\n" - contents += dedent("""\ +def make_worker() -> str: + return dedent("""\ from workers import WorkerEntrypoint class Default(WorkerEntrypoint): def test(self): pass """) - return contents -def make_snapshot( # noqa: PLR0913 +def make_snapshot( d: Path, outdir: Path, outprefix: str, compat_flags: list[str], - requirements: list[str], - imports: list[str], ) -> str: config_path = d / "config.capnp" - config_path.write_text(make_config(compat_flags, requirements)) + config_path.write_text(make_config(compat_flags)) worker_path = d / "worker.py" - worker_path.write_text(make_worker(imports)) - if imports: - snapshot_flag = "--python-save-snapshot" - else: - snapshot_flag = "--python-save-baseline-snapshot" + worker_path.write_text(make_worker()) if "WORKERD_BINARY" in environ: workerd = [environ["WORKERD_BINARY"]] @@ -123,11 +107,13 @@ def make_snapshot( # noqa: PLR0913 *workerd, "test", config_path, - snapshot_flag, + "--python-save-baseline-snapshot", "--pyodide-bundle-disk-cache-dir", d, "--pyodide-package-disk-cache-dir", d, + "--python-snapshot-dir", + d, "--experimental", ], ) @@ -144,42 +130,13 @@ def make_snapshot( # noqa: PLR0913 def make_baseline_snapshot( cache: Path, outdir: Path, compat_flags: list[str] ) -> list[tuple[str, str]]: - name, digest = make_snapshot(cache, outdir, "baseline", compat_flags, [], []) + name, digest = make_snapshot(cache, outdir, "baseline", compat_flags) return [ ("baseline_snapshot", name), ("baseline_snapshot_hash", digest), ] -def make_numpy_snapshot( - cache: Path, outdir: Path, compat_flags: list[str] -) -> list[tuple[str, str]]: - name, digest = make_snapshot( - cache, outdir, "package_snapshot_numpy", compat_flags, ["numpy"], ["numpy"] - ) - return [ - ("numpy_snapshot", name), - ("numpy_snapshot_hash", digest), - ] - - -def make_fastapi_snapshot( - cache: Path, outdir: Path, compat_flags: list[str] -) -> list[tuple[str, str]]: - name, digest = make_snapshot( - cache, - outdir, - "package_snapshot_fastapi", - compat_flags, - ["fastapi"], - ["fastapi", "pydantic"], - ) - return [ - ("fastapi_snapshot", name), - ("fastapi_snapshot_hash", digest), - ] - - def make_snapshots( cache: Path, outdir: Path, update_released: bool ) -> tuple[str, tuple[str, str]]: @@ -195,10 +152,6 @@ def make_snapshots( with timing(f"version {ver} snapshots"): with timing("baseline snapshot"): ver_info += make_baseline_snapshot(cache, outdir, compat_flags) - with timing("numpy snapshot"): - ver_info += make_numpy_snapshot(cache, outdir, compat_flags) - with timing("fastapi snapshot"): - ver_info += make_fastapi_snapshot(cache, outdir, compat_flags) res.append((ver, ver_info)) return res diff --git a/src/pyodide/types/emscripten.d.ts b/src/pyodide/types/emscripten.d.ts index 38114066ebd..083cf0e43e7 100644 --- a/src/pyodide/types/emscripten.d.ts +++ b/src/pyodide/types/emscripten.d.ts @@ -37,7 +37,7 @@ interface API { stdout?: (a: string) => void, stderr?: (a: string) => void ) => void; - version: '0.26.0a2' | '0.28.2'; + version: '0.26.0a2' | '0.28.2' | '314.0.0a1'; pyodide_base: { pyimport_impl: PyCallable; }; diff --git a/src/workerd/api/pyodide/pyodide.c++ b/src/workerd/api/pyodide/pyodide.c++ index 9ffc8272340..2f9a2007dda 100644 --- a/src/workerd/api/pyodide/pyodide.c++ +++ b/src/workerd/api/pyodide/pyodide.c++ @@ -539,6 +539,13 @@ kj::Maybe getPyodideLock(PythonSnapshotRelease::Reader pythonSnapsho } } + // From Pyodide 314 on, we don't use packages inside the lockfile. + // All packages used by the worker should come from PyPI and be bundled inside the worker. + // To avoid breaking existing workers, we return an empty lockfile if no packages are found. + if (pythonSnapshotRelease.getPackages().size() == 0) { + return kj::str("{\"packages\":{}}"); + } + return kj::none; } diff --git a/src/workerd/server/server.c++ b/src/workerd/server/server.c++ index 89a6857981b..9941474de7d 100644 --- a/src/workerd/server/server.c++ +++ b/src/workerd/server/server.c++ @@ -5890,14 +5890,17 @@ kj::Promise Server::preloadPython( // Fetch the Pyodide bundle. co_await server::fetchPyodideBundle(pythonConfig, kj::mv(version), network, timer); - // Preload Python packages. - KJ_IF_SOME(modulesSource, workerDef.source.variant.tryGet()) { - if (modulesSource.isPython) { - auto pythonRequirements = getPythonRequirements(modulesSource); - - // Store the packages in the package manager that is stored in the pythonConfig - co_await server::fetchPyodidePackages(pythonConfig, pythonConfig.pyodidePackageManager, - pythonRequirements, release, network, timer); + // Preload Python packages for older Pyodide versions that use package lock files. + auto pyodideVersion = release.getPyodide(); + if (pyodideVersion == "0.26.0a2" || pyodideVersion == "0.28.2") { + KJ_IF_SOME(modulesSource, + workerDef.source.variant.tryGet()) { + if (modulesSource.isPython) { + auto pythonRequirements = getPythonRequirements(modulesSource); + + co_await server::fetchPyodidePackages(pythonConfig, pythonConfig.pyodidePackageManager, + pythonRequirements, release, network, timer); + } } } } diff --git a/src/workerd/server/tests/python/BUILD.bazel b/src/workerd/server/tests/python/BUILD.bazel index 766631f0265..60aeea1df2d 100644 --- a/src/workerd/server/tests/python/BUILD.bazel +++ b/src/workerd/server/tests/python/BUILD.bazel @@ -34,6 +34,11 @@ py_wd_test("subdirectory") py_wd_test( "sdk", compat_date = "2026-01-01", + # FIXME: Pyodide 314 throws when the empty url is given to the response + python_flags = [ + "0.26.0a2", + "0.28.2", + ], ) gen_rust_import_tests() @@ -42,9 +47,23 @@ py_wd_test("undefined-handler") py_wd_test("vendor_dir") -py_wd_test("dont-snapshot-pyodide") +py_wd_test( + "dont-snapshot-pyodide", + python_flags = [ + "0.26.0a2", + "0.28.2", + # Uses builtin packages. Should be removed/updated when we drop supporting builtin packages. + ], +) -py_wd_test("filter-non-py-files") +py_wd_test( + "filter-non-py-files", + python_flags = [ + "0.26.0a2", + "0.28.2", + # Uses builtin packages. Should be removed/updated when we drop supporting builtin packages. + ], +) py_wd_test("durable-object") @@ -73,12 +92,20 @@ py_wd_test("default-class-with-legacy-global-handlers") py_wd_test( "fastapi", make_snapshot = False, + python_flags = [ + "0.26.0a2", + "0.28.2", + ], use_snapshot = "fastapi", ) py_wd_test( "numpy", make_snapshot = False, + python_flags = [ + "0.26.0a2", + "0.28.2", + ], use_snapshot = "numpy", ) diff --git a/src/workerd/server/tests/python/import_tests.bzl b/src/workerd/server/tests/python/import_tests.bzl index 29e2d72fa6f..c6696a069f1 100644 --- a/src/workerd/server/tests/python/import_tests.bzl +++ b/src/workerd/server/tests/python/import_tests.bzl @@ -84,8 +84,13 @@ def _gen_import_tests(to_test, python_version, pkg_skip_versions): skip_python_flags = skip_python_flags, ) +# TODO: Remove after built-in packages support is removed def gen_import_tests(*, pkg_skip_versions = {}): for python_version, info in BUNDLE_VERSION_INFO.items(): + if "packages" not in info: + # from Pyodide 314, we don't bundle any packages by default + # so we don't need to generate import tests for it + continue to_test = PYTHON_IMPORTS_TO_TEST[info["packages"]] _gen_import_tests(to_test, python_version, pkg_skip_versions = pkg_skip_versions) @@ -131,6 +136,9 @@ def _gen_rust_import_tests(python_version): python_version = python_version, ) +# Test using built-in packages, should be removed/updated after dropping built-in packages support def gen_rust_import_tests(): for python_version in BUNDLE_VERSION_INFO.keys(): + if "packages" not in BUNDLE_VERSION_INFO[python_version]: + continue _gen_rust_import_tests(python_version) diff --git a/src/workerd/server/tests/python/py_wd_test.bzl b/src/workerd/server/tests/python/py_wd_test.bzl index 651f8392cca..e3a84c174a1 100644 --- a/src/workerd/server/tests/python/py_wd_test.bzl +++ b/src/workerd/server/tests/python/py_wd_test.bzl @@ -30,15 +30,17 @@ def _py_wd_test_helper( templated_src = name_flag.replace("/", "-") + "@template" templated_src = "/".join(src.split("/")[:-1] + [templated_src]) - pkg_tag = BUNDLE_VERSION_INFO[python_flag]["packages"] - data = data + ["@all_pyodide_wheels_%s//:whls" % pkg_tag] - args = args + ["--pyodide-package-disk-cache-dir"] + pyodide_version = BUNDLE_VERSION_INFO[python_flag]["real_pyodide_version"] + + if pyodide_version in ("0.26.0a2", "0.28.2"): + pkg_tag = BUNDLE_VERSION_INFO[python_flag]["packages"] + data = data + ["@all_pyodide_wheels_%s//:whls" % pkg_tag] + args = args + ["--pyodide-package-disk-cache-dir"] - # +pyodide+ is a bzlmod canonical repository name - args.append("../+pyodide+all_pyodide_wheels_%s" % pkg_tag) + # +pyodide+ is a bzlmod canonical repository name + args.append("../+pyodide+all_pyodide_wheels_%s" % pkg_tag) load_snapshot = None - pyodide_version = BUNDLE_VERSION_INFO[python_flag]["real_pyodide_version"] if use_snapshot == "stacked": if pyodide_version == "0.26.0a2": use_snapshot = None @@ -49,9 +51,10 @@ def _py_wd_test_helper( if use_snapshot: version_info = BUNDLE_VERSION_INFO[python_flag] - snapshot = version_info[use_snapshot + "_snapshot"] - data = data + [":python_snapshots"] - load_snapshot = snapshot + snapshot = version_info.get(use_snapshot + "_snapshot", "") + if snapshot: + data = data + [":python_snapshots"] + load_snapshot = snapshot if load_snapshot and not make_snapshot: args += ["--python-load-snapshot", "load_snapshot.bin"] diff --git a/src/workerd/server/tests/python/vendor_dir/vendor_dir.wd-test b/src/workerd/server/tests/python/vendor_dir/vendor_dir.wd-test index 76bc5679d2e..57e2796f2fc 100644 --- a/src/workerd/server/tests/python/vendor_dir/vendor_dir.wd-test +++ b/src/workerd/server/tests/python/vendor_dir/vendor_dir.wd-test @@ -14,7 +14,6 @@ const unitTests :Workerd.Config = ( # This module below exercises a bug which caused our internal introspection.py # module to import it instead of the SDK module. See EW-9317 for more info. (name = "workers.py", pythonModule = embed "vendor/a.py"), - (name = "numpy", pythonRequirement = "") ], compatibilityFlags = [%PYTHON_FEATURE_FLAGS, "disable_python_no_global_handlers"], ) diff --git a/src/workerd/server/tests/python/vendor_dir_compat_flag/vendor_dir_compat_flag.wd-test b/src/workerd/server/tests/python/vendor_dir_compat_flag/vendor_dir_compat_flag.wd-test index 66efddd4415..407eaaed468 100644 --- a/src/workerd/server/tests/python/vendor_dir_compat_flag/vendor_dir_compat_flag.wd-test +++ b/src/workerd/server/tests/python/vendor_dir_compat_flag/vendor_dir_compat_flag.wd-test @@ -7,7 +7,6 @@ const unitTests :Workerd.Config = ( modules = [ (name = "worker.py", pythonModule = embed "worker.py"), (name = "vendor/a.py", pythonModule = embed "vendor/a.py"), - (name = "numpy", pythonRequirement = "") ], compatibilityFlags = [%PYTHON_FEATURE_FLAGS, "python_workers_force_new_vendor_path", "disable_python_no_global_handlers"], ) diff --git a/src/workerd/server/tests/python/vendor_pkg_tests/BUILD b/src/workerd/server/tests/python/vendor_pkg_tests/BUILD index 2a2f82aaa71..4162cd0ed02 100644 --- a/src/workerd/server/tests/python/vendor_pkg_tests/BUILD +++ b/src/workerd/server/tests/python/vendor_pkg_tests/BUILD @@ -3,7 +3,14 @@ load(":vendor_test.bzl", "vendored_py_wd_test") python_test_setup() -vendored_py_wd_test("fastapi") +vendored_py_wd_test( + "fastapi", + # FIXME: Pyodide 314 throws when the empty url is given to the response + skip_python_flags = [ + "314.0.0a1", + "development", + ], +) vendored_py_wd_test("beautifulsoup4")