Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/release-python-snapshots.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
secrets:
BAZEL_CACHE_KEY: ${{ secrets.BAZEL_CACHE_KEY }}
WORKERS_MIRROR_URL: ${{ secrets.WORKERS_MIRROR_URL }}
GOOGLESOURCE_COOKE: ${{ secrets.GOOGLESOURCE_COOKIE }}
GOOGLESOURCE_COOKIE: ${{ secrets.GOOGLESOURCE_COOKIE }}

build:
needs: [build-linux]
Expand Down
2 changes: 1 addition & 1 deletion build/deps/python.MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ pip.parse(
use_repo(pip, "py_deps", "v8_python_deps")

pyodide = use_extension("//build/deps:dep_pyodide.bzl", "pyodide")
use_repo(pyodide, "all_pyodide_wheels_20240829.4", "all_pyodide_wheels_20250808", "beautifulsoup4_src_0.26.0a2", "beautifulsoup4_src_0.28.2", "beautifulsoup4_src_development", "fastapi_src_0.26.0a2", "fastapi_src_0.28.2", "fastapi_src_development", "pyodide-0.26.0a2", "pyodide-0.28.2", "pyodide-lock_20240829.4.json", "pyodide-lock_20250808.json", "pyodide-snapshot-baseline-4569679fb.bin", "pyodide-snapshot-baseline-61eedf943.bin", "pyodide-snapshot-ew-py-package-snapshot_fastapi-v2.bin", "pyodide-snapshot-ew-py-package-snapshot_numpy-v2.bin", "pyodide-snapshot-package_snapshot_fastapi-a6ccb56fe.bin", "pyodide-snapshot-package_snapshot_numpy-60c9cb28e.bin", "pyodide-snapshot-snapshot_a6b652a95810783f5078b9a5dbd4a07c30718acb4ff724e82c25db7353dd7f2d.bin", "pyodide_0.26.0a2_2024-03-01_79.capnp.bin", "pyodide_0.28.2_2025-01-16_10.capnp.bin", "pyodide_dev.capnp.bin", "pytest-asyncio_src_0.26.0a2", "pytest-asyncio_src_0.28.2", "pytest-asyncio_src_development", "python-workers-runtime-sdk_src_0.26.0a2", "python-workers-runtime-sdk_src_0.28.2", "python-workers-runtime-sdk_src_development", "scipy_src_0.26.0a2", "shapely_src_0.28.2", "shapely_src_development")
use_repo(pyodide, "all_pyodide_wheels_20240829.4", "all_pyodide_wheels_20250808", "beautifulsoup4_src_0.26.0a2", "beautifulsoup4_src_0.28.2", "beautifulsoup4_src_314.0.0a1", "beautifulsoup4_src_development", "fastapi_src_0.26.0a2", "fastapi_src_0.28.2", "pyodide-0.26.0a2", "pyodide-0.28.2", "pyodide-314.0.0a1", "pyodide-lock_20240829.4.json", "pyodide-lock_20250808.json", "pyodide-snapshot-baseline-4569679fb.bin", "pyodide-snapshot-baseline-52636df0a.bin", "pyodide-snapshot-baseline-61eedf943.bin", "pyodide-snapshot-ew-py-package-snapshot_fastapi-v2.bin", "pyodide-snapshot-ew-py-package-snapshot_numpy-v2.bin", "pyodide-snapshot-package_snapshot_fastapi-a6ccb56fe.bin", "pyodide-snapshot-package_snapshot_numpy-60c9cb28e.bin", "pyodide-snapshot-snapshot_a6b652a95810783f5078b9a5dbd4a07c30718acb4ff724e82c25db7353dd7f2d.bin", "pyodide_0.26.0a2_2024-03-01_79.capnp.bin", "pyodide_0.28.2_2025-01-16_10.capnp.bin", "pyodide_314.0.0a1_2026-04-09_4.capnp.bin", "pyodide_dev.capnp.bin", "pytest-asyncio_src_0.26.0a2", "pytest-asyncio_src_0.28.2", "pytest-asyncio_src_314.0.0a1", "pytest-asyncio_src_development", "python-workers-runtime-sdk_src_0.26.0a2", "python-workers-runtime-sdk_src_0.28.2", "python-workers-runtime-sdk_src_314.0.0a1", "python-workers-runtime-sdk_src_development", "scipy_src_0.26.0a2", "shapely_src_0.28.2")
29 changes: 23 additions & 6 deletions build/python_metadata.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ PYODIDE_VERSIONS = [
"version": "0.28.2",
"sha256": "c9f6dd067d119e50850849f7428e3c636ecbc2684a0d2ff992f3bd48a1062b6c",
},
{
"version": "314.0.0a1",
"sha256": "7cceacea2efb493f30667192f0380e1fc6955c38fe34897b2d5be98023758073",
},
]

# This is the list of all the package metadata that we use.
Expand All @@ -34,6 +38,8 @@ PYODIDE_VERSIONS = [
_package_lockfiles = [
PACKAGES_20240829_4,
PACKAGES_20250808,
# As of Pyodide 314, we don't bundle any packages by default
# So this list does not need to be updated anymore.
]

# The below is a list of pyodide-lock.json files for each package bundle version that we support.
Expand Down Expand Up @@ -69,6 +75,7 @@ def _add_integrity(entry):
def _make_vendored_packages(entry):
if entry["name"] == "development":
return

vendor_tests = {}
for e in entry["vendored_packages_for_tests"]:
vendor_tests[e["name"]] = e
Expand Down Expand Up @@ -158,6 +165,7 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([
},
{
"name": "0.28.2",
"released": True,
"pyodide_version": "0.28.2",
"pyodide_date": "2025-01-16",
"packages": PACKAGES_20250808,
Expand All @@ -181,11 +189,6 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([
"abi": "3.13",
"sha256": "955091f1bd2eb33255ff2633df990bedc96e2f6294e78f2b416078777394f942",
},
# {
# "name": "scipy",
# "abi": "3.13",
# "sha256": "4f1b6fc179bd5c6d3de68abc4aa9fca2aaecd09c5c8d357c2ecfedce7d621f3d",
# },
{
"name": "shapely",
"abi": "3.13",
Expand All @@ -194,7 +197,21 @@ BUNDLE_VERSION_INFO = _make_bundle_version_info([
],
},
{
"real_pyodide_version": "0.28.2",
"name": "314.0.0a1",
"pyodide_version": "314.0.0a1",
"pyodide_date": "2026-04-09",
"backport": "4",
"integrity": "sha256-T8JMW0IoonHdZTqlvHgkQrHT6ndyXXfnbbUP3UO15ag=",
"flag": "pythonWorkersDevPyodide",
"enable_flag_name": "python_workers_development",
"emscripten_version": "5.0.3",
"python_version": "3.14.2",
"baseline_snapshot": "baseline-52636df0a.bin",
"baseline_snapshot_hash": "52636df0a566fe69bc762796c458603bc69aad434b59e16e1d46c9bc8a2f04bf",
"vendored_packages_for_tests": VENDORED_VERSION_INDEPENDENT,
},
{
"real_pyodide_version": "314.0.0a1",
Comment thread
ryanking13 marked this conversation as resolved.
"name": "development",
"pyodide_version": "dev",
"pyodide_date": "dev",
Expand Down
126 changes: 90 additions & 36 deletions src/pyodide/helpers.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@ def _copy_and_capnp_embed(src):
def _fmt_python_snapshot_release(
pyodide_version,
pyodide_date,
packages,
backport,
baseline_snapshot_hash,
flag,
real_pyodide_version,
packages = "",
**_kwds):
content = ", ".join(
[
Expand Down Expand Up @@ -162,20 +162,9 @@ import {
} from "pyodide-internal:pool/builtin_wrappers";
"""

# pyodide.asm.js patches
# pyodide.asm.mjs patches
# TODO: all of these should be fixed by linking our own Pyodide or by upstreaming.
_REPLACEMENTS = [
[
# Convert pyodide.asm.js into an es6 module.
# When we link our own we can pass `-sES6_MODULE` to the linker and it will do this for us
# automatically.
"var _createPyodideModule",
_PRELUDE + "export const _createPyodideModule",
],
[
"globalThis._createPyodideModule = _createPyodideModule;",
"",
],
_REPLACEMENTS_COMMON = [
[
"new WebAssembly.Module",
"newWasmModule",
Expand Down Expand Up @@ -227,15 +216,6 @@ _REPLACEMENTS = [
"getMemory(",
"Module.getMemoryPatched(Module, libName, ",
],
# to fix RPC, applies https://github.com/pyodide/pyodide/commit/8da1f38f7
[
"nullToUndefined(func.apply(",
"nullToUndefined(patchedApplyFunc(API, func, ",
],
[
"nullToUndefined(Function.prototype.apply.apply",
"nullToUndefined(API.config.jsglobals.Function.prototype.apply.apply",
],
[
"function _PyEM_CountFuncParams(func){",
"function _PyEM_CountFuncParams(func){ return patched_PyEM_CountFuncParams(Module, func);",
Expand All @@ -253,13 +233,74 @@ _REPLACEMENTS = [
],
]

def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_js = None, python_stdlib_zip = None, emscripten_setup_override = None):
_REPLACEMENTS = {
"0.26.0a2": _REPLACEMENTS_COMMON + [
# for 0.28.2 or earlier, pyodide.asm.js was a commonjs module
[
# Convert pyodide.asm.js into an es6 module.
# When we link our own we can pass `-sES6_MODULE` to the linker and it will do this for us
# automatically.
"var _createPyodideModule",
_PRELUDE + "export const _createPyodideModule",
],
[
"globalThis._createPyodideModule = _createPyodideModule;",
"",
],
# to fix RPC, applies https://github.com/pyodide/pyodide/commit/8da1f38f7
[
"nullToUndefined(func.apply(",
"nullToUndefined(patchedApplyFunc(API, func, ",
],
[
"nullToUndefined(Function.prototype.apply.apply",
"nullToUndefined(API.config.jsglobals.Function.prototype.apply.apply",
],
],
"0.28.2": _REPLACEMENTS_COMMON + [
# for 0.28.2 or earlier, pyodide.asm.js was a commonjs module
[
# Convert pyodide.asm.js into an es6 module.
# When we link our own we can pass `-sES6_MODULE` to the linker and it will do this for us
# automatically.
"var _createPyodideModule",
_PRELUDE + "export const _createPyodideModule",
],
[
"globalThis._createPyodideModule = _createPyodideModule;",
"",
],
# to fix RPC, applies https://github.com/pyodide/pyodide/commit/8da1f38f7
[
"nullToUndefined(func.apply(",
"nullToUndefined(patchedApplyFunc(API, func, ",
],
[
"nullToUndefined(Function.prototype.apply.apply",
"nullToUndefined(API.config.jsglobals.Function.prototype.apply.apply",
],
],
"314.0.0a1": _REPLACEMENTS_COMMON + [
# for 314 or later, pyodide.asm.mjs is es6 module
[
"export default _createPyodideModule;",
# still expose _createPyodideModule for compatibility (import { _createPyodideModule })
_PRELUDE + "export default _createPyodideModule; export { _createPyodideModule };",
],
],
}

def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_mjs = None, python_stdlib_zip = None, emscripten_setup_override = None):
pyodide_package = "@pyodide-%s//" % version
if not pyodide_asm_wasm:
pyodide_asm_wasm = pyodide_package + ":pyodide/pyodide.asm.wasm"

if not pyodide_asm_js:
pyodide_asm_js = pyodide_package + ":pyodide/pyodide.asm.js"
if not pyodide_asm_mjs:
if version in ("0.26.0a2", "0.28.2"):
pyodide_asm_mjs = pyodide_package + ":pyodide/pyodide.asm.js"
else:
# for 314 or later, pyodide.asm.mjs is es6 module
pyodide_asm_mjs = pyodide_package + ":pyodide/pyodide.asm.mjs"

if not python_stdlib_zip:
python_stdlib_zip = pyodide_package + ":pyodide/python_stdlib.zip"
Expand All @@ -269,33 +310,42 @@ def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_js = None, p
_copy_to_generated(python_stdlib_zip, version, out_name = "python_stdlib.zip")

expand_template(
name = "pyodide.asm.js@rule@" + version,
out = _out_path("pyodide.asm.js", version),
substitutions = dict(_REPLACEMENTS),
template = pyodide_asm_js,
name = "pyodide.asm.mjs@rule@" + version,
out = _out_path("pyodide.asm.mjs", version),
substitutions = dict(_REPLACEMENTS[version]),
template = pyodide_asm_mjs,
)

js_file(
name = "pyodide.asm.js@rule_js@" + version,
srcs = [_out_path("pyodide.asm.js", version)],
deps = ["pyodide.asm.js@rule@" + version],
name = "pyodide.asm.mjs@rule_js@" + version,
srcs = [_out_path("pyodide.asm.mjs", version)],
deps = ["pyodide.asm.mjs@rule@" + version],
)

if emscripten_setup_override:
_copy_to_generated(out_name = "emscriptenSetup.js", name = "emscriptenSetup", src = emscripten_setup_override, version = version)
else:
expand_template(
name = "esbuild.config.mjs@" + version,
out = _out_path("esbuild.config.mjs", version),
substitutions = {
"%PYODIDE_VERSION%": version,
},
template = "internal/pool/esbuild.config.mjs",
)

esbuild(
name = "emscriptenSetup@" + version,
# exclude emscriptenSetup from source set so that rules_ts won't also try to create a JS output
# for it. The file is provided in entry_point instead.
srcs = native.glob([
"internal/pool/*.ts",
], exclude = ["internal/pool/emscriptenSetup.ts"]) + [
_out_path("pyodide.asm.js", version),
_out_path("pyodide.asm.mjs", version),
"internal/util.ts",
"internal/const.ts",
],
config = "internal/pool/esbuild.config.mjs",
config = _out_path("esbuild.config.mjs", version),
entry_point = "internal/pool/emscriptenSetup.ts",
external = [
"child_process",
Expand All @@ -312,11 +362,15 @@ def _python_bundle(version, *, pyodide_asm_wasm = None, pyodide_asm_js = None, p
"node:url",
"node:vm",
"internal:unsafe-eval",
"node:stream",
"node:tls",
"node:net",
"node:module",
],
format = "esm",
output = _out_path("emscriptenSetup.js", version),
target = "esnext",
deps = ["pyodide.asm.js@rule_js@" + version],
deps = ["pyodide.asm.mjs@rule_js@" + version],
)

import_name = "pyodideRuntime"
Expand Down
1 change: 1 addition & 0 deletions src/pyodide/internal/const.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@
export const PyodideVersion = {
V0_26_0a2: '0.26.0a2',
V0_28_2: '0.28.2',
V314_0_0a1: '314.0.0a1',
} as const;
30 changes: 25 additions & 5 deletions src/pyodide/internal/pool/emscriptenSetup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import {
finishSetup,
} from 'pyodide-internal:pool/builtin_wrappers';

import { getSentinelImport } from 'pyodide-internal:pool/sentinel';
import { getJsvErrorImport } from 'pyodide-internal:pool/sentinel';

/**
* A preRun hook. Make sure environment variables are visible at runtime.
Expand Down Expand Up @@ -110,7 +110,7 @@ function getPrepareFileSystem(pythonStdlib: ArrayBuffer): PreRunHook {
function getInstantiateWasm(
pyodideWasmModule: WebAssembly.Module
): EmscriptenSettings['instantiateWasm'] {
const sentinelImportPromise = getSentinelImport();
const jsvErrorImportPromise = getJsvErrorImport();
return function instantiateWasm(
wasmImports: WebAssembly.Imports,
successCallback: (
Expand All @@ -119,8 +119,20 @@ function getInstantiateWasm(
) => void
): WebAssembly.Exports {
(async function (): Promise<void> {
wasmImports.sentinel = await sentinelImportPromise;
// Instantiate pyodideWasmModule with wasmImports
const { Jsv_GetError_import, JsvError_Check } =
await jsvErrorImportPromise;

// Pyodide <= 0.28.2: These were named create_sentinel and is_sentinel
// Pyodide >= 314: These are renamed to Jsv_GetError_import and JsvError_Check and moved to env namespace
wasmImports.sentinel = {
create_sentinel: Jsv_GetError_import,
is_sentinel: JsvError_Check,
};
const env = wasmImports.env;
if (env) {
env.Jsv_GetError_import = Jsv_GetError_import;
env.JsvError_Check = JsvError_Check;
}
const instance = await WebAssembly.instantiate(
pyodideWasmModule,
wasmImports
Expand Down Expand Up @@ -168,7 +180,15 @@ function getEmscriptenSettings(
(res) => (config.resolveLockFilePromise = res)
);
}
const API = { config, lockFilePromise };
const API = {
config,
lockFilePromise,
runtimeEnv: {
IN_WORKERD: true,
IN_BROWSER: true,
IN_BROWSER_MAIN_THREAD: true,
},
};
let resolveReadyPromise: (mod: Module) => void;
let rejectReadyPromise: (e: any) => void = () => {};
const readyPromise: Promise<Module> = new Promise((res, rej) => {
Expand Down
16 changes: 8 additions & 8 deletions src/pyodide/internal/pool/esbuild.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ const pyodideRootDir = dirname(
dirname(dirname(fileURLToPath(import.meta.url)))
);

const pyodideVersion = '%PYODIDE_VERSION%';

let resolvePlugin = {
name: 'pyodide-internal',
setup(build) {
Expand All @@ -14,15 +16,13 @@ let resolvePlugin = {
let rest = args.path.split(':')[1];
let path;
if (rest.startsWith('generated')) {
// I couldn't figure out how to pass down the version, so instead we'll look through the
// directories in `pyodideRootDir` and find one that starts with a 0. This will work until
// Pyodide has a 1.0 release.
const dir = readdirSync(pyodideRootDir).filter((x) =>
x.startsWith('0')
)[0];
path = join(pyodideRootDir, dir, rest);
path = join(pyodideRootDir, pyodideVersion, rest);
if (!existsSync(path)) {
path += '.js';
if (existsSync(path + '.js')) {
path += '.js';
} else {
path += '.mjs';
}
}
} else {
path = join(pyodideRootDir, 'internal', rest);
Expand Down
Loading
Loading