From 840bf33eed04020b0f8eccad1ab7a0e8f0da06b3 Mon Sep 17 00:00:00 2001 From: Julius Jurgelenas Date: Fri, 14 Aug 2026 22:15:04 +0300 Subject: [PATCH] Add Lua bytecode compilation and production package builds Scripts shipped as source are compiled by the radio on first load, which costs RAM and startup time, and there was no way to produce a release artifact. Wrap EdgeTX's edgetx-luac WebAssembly module (upstream PR #7275) in a WAMR runtime and build four things on top of it: - dev luac: compile a single script - dev build --out: a package directory with .luac next to every .lua, installable via pkg install - dev build --compress: an SD-card zip laid out at install destinations with no manifest, for artifact storage - pkg install/update --pre-compile: bytecode installed alongside sources Compilation never touches the SD card. --pre-compile runs the same build into a local TempDir and copies the finished tree across in one pass, so a script that fails to compile aborts before the card, the store, or the file list is touched. Sources ship next to the bytecode because the radio still discovers widgets by their .lua (EdgeTX#5230), and generated .luac files take their source's timestamp so the radio prefers them instead of recompiling. The module is downloaded on first use and cached; its output is byte-identical to the .luac the simulator and radio generate themselves, verified against six ELRS widget scripts. --- Makefile | 2 +- README.md | 54 ++++ docs/Manifest.md | 2 +- src/cli/dev.rs | 270 ++++++++++++++++++ src/cli/mod.rs | 24 ++ src/cli/pkg.rs | 70 ++++- src/luac/mod.rs | 388 ++++++++++++++++++++++++++ src/main.rs | 1 + src/manifest/mod.rs | 16 ++ src/packages/build.rs | 583 +++++++++++++++++++++++++++++++++++++++ src/packages/install.rs | 180 ++++++++++-- src/packages/mod.rs | 11 + src/packages/remove.rs | 49 +++- src/packages/transfer.rs | 101 ++++++- src/packages/update.rs | 24 +- tests/luac_compile.rs | 315 +++++++++++++++++++++ 16 files changed, 2047 insertions(+), 43 deletions(-) create mode 100644 src/luac/mod.rs create mode 100644 src/packages/build.rs create mode 100644 tests/luac_compile.rs diff --git a/Makefile b/Makefile index 03fb49c..8018bba 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ test-verbose: cargo test -- --nocapture test-integration: - cargo test --test simulator_script -- --ignored + cargo test --test simulator_script --test luac_compile -- --ignored lint: cargo clippy -- -D warnings diff --git a/README.md b/README.md index 16e5678..b6730dd 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ A command-line tool for managing Lua script packages on EdgeTX radios - and for - **Scaffold scripts** -generate boilerplate for tools, widgets, telemetry, functions, mixes, and libraries - **Package manifests** -`edgetx.yml` defines your scripts, dependencies, file layout, and exclusions - **Simulator** -run an EdgeTX simulator locally with live-reload, headless mode, and Lua test scripts +- **Bytecode compilation** -compile Lua to `.luac` with EdgeTX's own compiler, and build production packages or SD card archives - **Cross-platform** -Linux, macOS, and Windows with platform-specific radio detection ## Installation @@ -167,6 +168,10 @@ edgetx-cli pkg install Org/Repo --path edgetx.c480x272.yml | `--eject` | `false` | Safely unmount and power off the radio after install | | `--dry-run` | `false` | Show what would be installed without writing anything | | `--dev` | `false` | Include development dependencies | +| `--pre-compile` | `false` | Compile Lua scripts and install the bytecode alongside the sources | +| `--keep-debug` | `false` | Keep debug info in the compiled bytecode (requires `--pre-compile`) | + +**Pre-compiling:** `--pre-compile` builds the package in a local staging directory, compiles a `.luac` next to every `.lua`, and copies the finished tree to the card in one pass — nothing is compiled against the SD card itself. The radio then loads the bytecode instead of compiling on first use. A script that fails to compile fails the install before anything is written. See [`dev luac`](#dev-luac-file) for the compiler. **Version resolution:** When no `@version` is specified, the CLI queries the remote repository for all tags and installs the latest [semver](https://semver.org/)-compatible tag (e.g. `v2.1.0`). If no semver tags exist, the default branch HEAD is used. @@ -197,6 +202,8 @@ edgetx-cli pkg update --all | `--eject` | `false` | Safely unmount radio after update | | `--dry-run` | `false` | Show what would be updated without writing anything | | `--dev` | `false` | Include development dependencies (overrides the stored install preference)| +| `--pre-compile` | `false` | Compile Lua scripts and install the bytecode alongside the sources | +| `--keep-debug` | `false` | Keep debug info in the compiled bytecode (requires `--pre-compile`) | ### `pkg remove ` @@ -286,6 +293,10 @@ Checks each installed package against its remote source and reports available up ```sh edgetx-cli pkg install . --eject ``` +6. **Build a release artifact:** + ```sh + edgetx-cli dev build --out dist --compress + ``` ### `dev init [name]` @@ -340,6 +351,49 @@ edgetx-cli dev sync --src-dir ./my-project /path/to/edgetx-sdcard | `--src-dir` | `.` | Source directory containing `edgetx.yml` | | `--no-dev` | `false` | Exclude development dependencies | +### `dev build` + +Build a production copy of a package with compiled bytecode, ready to publish to artifact storage. + +By default the output is a package directory — `edgetx.yml` plus your content, with a `.luac` next to every `.lua` — which installs like any other local package. With `--compress` the output is instead a `.zip` laid out for the SD card: files sit at their install destinations, there is no manifest, and it extracts straight onto the card root. + +Sources ship alongside the bytecode because the radio still discovers some script types (widgets in particular) by their `.lua`. + +```sh +edgetx-cli dev build --out dist +edgetx-cli dev build --out dist --path edgetx.c480x272.yml +edgetx-cli dev build --out artifacts --compress +edgetx-cli dev build --out artifacts --compress --name my-tool-v1.2.3 +``` + +| Flag | Default | Description | +|----------------|---------|-----------------------------------------------------------------| +| `--out` | | Output directory (required) | +| `--src-dir` | `.` | Source directory containing `edgetx.yml` | +| `--path` | | Manifest file or subdirectory to build (for packages with variants) | +| `--compress` | `false` | Create a `.zip` archive for the SD card instead of a package directory | +| `--name` | | Archive base name (default: the package id with `/` replaced by `-`) | +| `--keep-debug` | `false` | Keep debug info in the compiled bytecode | +| `--force` | `false` | Replace an existing build in the output directory | + +Development-only content is always excluded. Since the manifest carries no version, use `--name` to stamp a version into release artifacts. + +### `dev luac ` + +Compile a single Lua script to EdgeTX bytecode. + +```sh +edgetx-cli dev luac SCRIPTS/TOOLS/MyTool/main.lua +edgetx-cli dev luac main.lua -o build/main.luac --keep-debug +``` + +| Flag | Default | Description | +|----------------|---------|----------------------------------------------------| +| `--output`/`-o`| input with a `.luac` extension | Output path | +| `--keep-debug` | `false` | Keep debug info in the bytecode (stripping is the default, matching the radio) | + +The compiler is EdgeTX's own [`edgetx-luac`](https://github.com/EdgeTX/edgetx/pull/7275) WebAssembly module, downloaded on first use and cached under your cache directory (`edgetx-cli/luac/`). Delete that file to pick up a newer build. Its output is byte-identical to the bytecode the radio and simulator produce themselves. + ### `dev simulator` Run the EdgeTX simulator. When run from a directory containing `edgetx.yml`, the package is automatically installed into the simulator's SD card and file changes are live-synced. diff --git a/docs/Manifest.md b/docs/Manifest.md index f88c91d..9d02964 100644 --- a/docs/Manifest.md +++ b/docs/Manifest.md @@ -83,7 +83,7 @@ themes: - `exclude` takes glob patterns to skip during copy (e.g., `["*.luac", "presets.txt"]`) - `themes` installs to `THEMES//` on the SD card — typically a directory with `theme.yml`, `logo.png`, and resolution-specific backgrounds. Themes require a color LCD; set `package.capabilities.display.type: colorlcd` - `source_dir` is relative to the manifest file; all `path` values are relative to the source root and must use `/` as the separator (never `\`), since they represent paths on a FAT32 SD card -- `binary: true` disables the default `*.luac` exclusion, allowing compiled bytecode to be installed +- `binary: true` disables the default `*.luac` exclusion, allowing compiled bytecode to be installed. `dev build` sets it in the manifest it emits, since a built package ships bytecode - `dev: true` marks a content item as a development dependency - it is excluded from `pkg install` and `pkg update` unless `--dev` is passed, but included by default in `dev sync` (use `--no-dev` to exclude). A non-dev item cannot depend on a dev library ### Source vs destination diff --git a/src/cli/dev.rs b/src/cli/dev.rs index 3e147bf..53d0b07 100644 --- a/src/cli/dev.rs +++ b/src/cli/dev.rs @@ -1,4 +1,7 @@ +use crate::luac::LuaCompiler; use crate::manifest; +use crate::packages; +use crate::radio; use crate::scaffold; use crate::simulator; use anyhow::{Context, Result, bail}; @@ -44,6 +47,55 @@ pub enum DevCommands { Sync(SyncArgs), /// Run the EdgeTX WASM simulator Simulator(SimulatorArgs), + /// Compile a Lua script to EdgeTX bytecode (.luac) + Luac(LuacArgs), + /// Build a production package with compiled bytecode + Build(BuildArgs), +} + +#[derive(Args)] +pub struct BuildArgs { + /// Output directory for the build + #[arg(long)] + out: PathBuf, + + /// Source directory containing edgetx.yml + #[arg(long, default_value = ".")] + src_dir: PathBuf, + + /// Manifest file or subdirectory to build (for packages with variants) + #[arg(long)] + path: Option, + + /// Create a .zip archive for the SD card instead of a package directory + #[arg(long)] + compress: bool, + + /// Archive base name (default: the package id with '/' replaced by '-') + #[arg(long)] + name: Option, + + /// Keep debug info in the compiled bytecode + #[arg(long)] + keep_debug: bool, + + /// Replace an existing build in the output directory + #[arg(long)] + force: bool, +} + +#[derive(Args)] +pub struct LuacArgs { + /// Lua script to compile + file: PathBuf, + + /// Output path (default: the input with a .luac extension) + #[arg(short, long)] + output: Option, + + /// Keep debug info in the compiled bytecode + #[arg(long)] + keep_debug: bool, } #[derive(Args)] @@ -147,7 +199,225 @@ pub fn dispatch(command: DevCommands) -> Result<()> { DevCommands::Scaffold(args) => run_scaffold(args), DevCommands::Sync(args) => run_sync(args), DevCommands::Simulator(args) => run_simulator(args), + DevCommands::Luac(args) => run_luac(args), + DevCommands::Build(args) => run_build(args), + } +} + +/// Default output path for a compiled script: `main.lua` -> `main.luac`. +fn default_luac_path(input: &Path) -> PathBuf { + input.with_extension("luac") +} + +fn run_luac(args: LuacArgs) -> Result<()> { + let source = + std::fs::read(&args.file).with_context(|| format!("reading {}", args.file.display()))?; + + let output = args + .output + .clone() + .unwrap_or_else(|| default_luac_path(&args.file)); + + let mut compiler = super::load_compiler(!args.keep_debug)?; + let bytecode = compiler + .compile(&source) + .with_context(|| format!("compiling {}", args.file.display()))?; + + std::fs::write(&output, &bytecode).with_context(|| format!("writing {}", output.display()))?; + + println!( + " {} Compiled {} -> {} ({} bytes)", + console::style("✓").green(), + args.file.display(), + output.display(), + bytecode.len() + ); + + Ok(()) +} + +fn run_build(args: BuildArgs) -> Result<()> { + let src_dir = std::fs::canonicalize(&args.src_dir) + .with_context(|| format!("resolving source directory {}", args.src_dir.display()))?; + + let sub_path = args.path.clone().unwrap_or_default(); + let (m, manifest_dir) = manifest::load_with_sub_path(&src_dir, &sub_path)?; + + if m.has_variants() { + let paths: Vec<&str> = m.package.variants.iter().map(|v| v.path.as_str()).collect(); + bail!( + "{:?} selects between variants, so there is nothing to build directly.\n \ + Build one with --path : {}", + m.package.display_name(), + paths.join(", ") + ); } + + std::fs::create_dir_all(&args.out) + .with_context(|| format!("creating output directory {}", args.out.display()))?; + let out_dir = std::fs::canonicalize(&args.out) + .with_context(|| format!("resolving output directory {}", args.out.display()))?; + + // A build writes into its output tree, so overlapping it with the sources + // would have the copy walking over its own output. + for root in m.source_roots(&manifest_dir) { + if out_dir.starts_with(&root) { + bail!( + "output directory {} is inside the source directory {}", + out_dir.display(), + root.display() + ); + } + } + + let base_name = args + .name + .clone() + .unwrap_or_else(|| packages::build::archive_base_name(&m.package.id)); + + // A zip is built through a staging directory next to it, which compress_dir + // consumes on success. + let (target, staging) = if args.compress { + ( + out_dir.join(format!("{base_name}.zip")), + out_dir.join(&base_name), + ) + } else { + (out_dir.clone(), out_dir.clone()) + }; + + if !args.force && build_target_exists(&target)? { + bail!( + "{} already exists -- pass --force to replace it", + target.display() + ); + } + if args.force { + clear_build_target(&target)?; + } + if args.compress { + clear_build_target(&staging)?; + } + + println!(); + println!(" {}", console::style(m.package.display_name()).bold()); + println!(); + + let mut compiler = super::load_compiler(!args.keep_debug)?; + + let bar = ProgressBar::new(packages::build::count_steps(&m, &manifest_dir, false) as u64); + bar.set_style( + ProgressStyle::with_template("{spinner:.green} [{bar:40.cyan/blue}] {pos}/{len} {msg}") + .unwrap(), + ); + bar.set_message("Building"); + + let tick = |path: &str| { + if let Some(name) = Path::new(path).file_name() { + bar.set_message(name.to_string_lossy().to_string()); + } + bar.inc(1); + }; + + let layout = if args.compress { + packages::build::BuildLayout::SdOverlay + } else { + packages::build::BuildLayout::Package + }; + + let result = packages::build::build_package( + &packages::build::BuildOptions { + manifest: &m, + manifest_dir: &manifest_dir, + out_dir: &staging, + include_dev: false, + layout, + }, + &mut compiler, + &mut |p: &str| tick(p), + &mut |p: &str| tick(p), + ); + bar.finish_and_clear(); + let result = result?; + + if args.compress { + let zip_total = radio::backup::count_all_files(&staging); + let zip_bar = ProgressBar::new(zip_total as u64); + zip_bar.set_style( + ProgressStyle::with_template("{spinner:.green} [{bar:40.cyan/blue}] {pos}/{len} {msg}") + .unwrap(), + ); + zip_bar.set_message("Compressing"); + + radio::backup::compress_dir(&staging, &target, |rel| { + if let Some(name) = Path::new(rel).file_name() { + zip_bar.set_message(name.to_string_lossy().to_string()); + } + zip_bar.inc(1); + })?; + zip_bar.finish_and_clear(); + } + + println!( + " {} Built {} file(s), compiled {} script(s)", + console::style("✓").green(), + result.files_copied, + result.scripts_compiled + ); + println!( + " {} Output: {}", + console::style("ℹ").blue(), + target.display() + ); + if args.compress { + println!( + " {} Extract the archive onto the SD card root", + console::style("ℹ").blue() + ); + } else { + println!( + " {} Install it with: edgetx-cli pkg install {}", + console::style("ℹ").blue(), + target.display() + ); + } + println!(); + + Ok(()) +} + +/// Whether a previous build is already sitting at `target`. +fn build_target_exists(target: &Path) -> Result { + if target.is_file() { + return Ok(true); + } + if !target.is_dir() { + return Ok(false); + } + let mut entries = + std::fs::read_dir(target).with_context(|| format!("reading {}", target.display()))?; + Ok(entries.next().is_some()) +} + +/// Remove a previous build so the new one cannot inherit stale files. +fn clear_build_target(target: &Path) -> Result<()> { + if target.is_dir() { + for entry in std::fs::read_dir(target) + .with_context(|| format!("reading {}", target.display()))? + .flatten() + { + let path = entry.path(); + let removed = if path.is_dir() { + std::fs::remove_dir_all(&path) + } else { + std::fs::remove_file(&path) + }; + removed.with_context(|| format!("removing {}", path.display()))?; + } + } else if target.is_file() { + std::fs::remove_file(target).with_context(|| format!("removing {}", target.display()))?; + } + Ok(()) } fn run_init(args: InitArgs) -> Result<()> { diff --git a/src/cli/mod.rs b/src/cli/mod.rs index f6f06ef..9bbec9b 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -52,6 +52,30 @@ pub enum Commands { Eject(eject::EjectArgs), } +/// Load the EdgeTX Lua compiler, downloading it on first use. +/// +/// Progress is only reported when a download actually happens, so the common +/// cached path stays quiet. +pub(crate) fn load_compiler(strip: bool) -> anyhow::Result { + use std::sync::atomic::{AtomicBool, Ordering}; + + let downloading = AtomicBool::new(false); + let result = crate::luac::Compiler::from_cache_or_download(strip, |downloaded, total| { + downloading.store(true, Ordering::Relaxed); + if total > 0 { + let pct = downloaded as f64 / total as f64 * 100.0; + eprint!("\r Downloading the EdgeTX Lua compiler... {pct:.0}%"); + } + }); + + if downloading.load(Ordering::Relaxed) { + eprintln!(); + println!(" {} Lua compiler ready", console::style("✓").green()); + } + + Ok(result?) +} + pub fn dispatch(cli: Cli) -> anyhow::Result<()> { match cli.command { Commands::Backup(args) => backup::run(args), diff --git a/src/cli/pkg.rs b/src/cli/pkg.rs index e4038ef..c684f0c 100644 --- a/src/cli/pkg.rs +++ b/src/cli/pkg.rs @@ -4,6 +4,7 @@ use indicatif::{ProgressBar, ProgressStyle}; use std::path::{Path, PathBuf}; use std::time::Duration; +use crate::luac::LuaCompiler; use crate::packages; use crate::radio; use crate::source::{PackageRef, resolve}; @@ -51,6 +52,14 @@ pub struct InstallArgs { /// Manifest file or subdirectory within the repo #[arg(long)] path: Option, + + /// Compile Lua scripts and install the bytecode alongside the sources + #[arg(long)] + pre_compile: bool, + + /// Keep debug info in the compiled bytecode + #[arg(long, requires = "pre_compile")] + keep_debug: bool, } #[derive(Args)] @@ -81,6 +90,14 @@ pub struct UpdateArgs { /// Include development dependencies #[arg(long)] dev: bool, + + /// Compile Lua scripts and install the bytecode alongside the sources + #[arg(long)] + pre_compile: bool, + + /// Keep debug info in the compiled bytecode + #[arg(long, requires = "pre_compile")] + keep_debug: bool, } #[derive(Args)] @@ -230,8 +247,16 @@ fn run_install(args: InstallArgs) -> Result<()> { } println!(); + // A dry run writes nothing, so there is nothing to compile for it either. + let pre_compile = args.pre_compile && !args.dry_run; + let mut compiler = if pre_compile { + Some(super::load_compiler(!args.keep_debug)?) + } else { + None + }; + // Progress bar - let total_files = cmd.total_files(); + let total_files = cmd.total_files(pre_compile); let bar = ProgressBar::new(total_files as u64); bar.set_style( ProgressStyle::with_template("{spinner:.green} [{bar:40.cyan/blue}] {pos}/{len} {msg}") @@ -239,12 +264,16 @@ fn run_install(args: InstallArgs) -> Result<()> { ); bar.set_message("Installing"); - let result = cmd.execute(args.dry_run, |dest| { - if let Some(name) = Path::new(dest).file_name() { - bar.set_message(name.to_string_lossy().to_string()); - } - bar.inc(1); - })?; + let result = cmd.execute( + args.dry_run, + compiler.as_mut().map(|c| c as &mut dyn LuaCompiler), + |dest| { + if let Some(name) = Path::new(dest).file_name() { + bar.set_message(name.to_string_lossy().to_string()); + } + bar.inc(1); + }, + )?; bar.finish_and_clear(); println!(); @@ -301,6 +330,15 @@ fn run_update(args: UpdateArgs) -> Result<()> { let mut store = packages::store::PackageStore::load(sd_root.clone())?; let targets = store.update_targets(&query, args.all)?; + // A dry run writes nothing, so there is nothing to compile for it either. + // One compiler serves every target in an --all run. + let pre_compile = args.pre_compile && !args.dry_run; + let mut compiler = if pre_compile { + Some(super::load_compiler(!args.keep_debug)?) + } else { + None + }; + println!(); for target in &targets { let include_dev = if args.package.is_some() { @@ -318,7 +356,7 @@ fn run_update(args: UpdateArgs) -> Result<()> { store, )?; - let total_files = cmd.total_files(); + let total_files = cmd.total_files(pre_compile); let bar = ProgressBar::new(total_files as u64); bar.set_style( ProgressStyle::with_template("{spinner:.green} [{bar:40.cyan/blue}] {pos}/{len} {msg}") @@ -326,12 +364,16 @@ fn run_update(args: UpdateArgs) -> Result<()> { ); bar.set_message("Updating"); - let result = cmd.execute(args.dry_run, |dest| { - if let Some(name) = Path::new(dest).file_name() { - bar.set_message(name.to_string_lossy().to_string()); - } - bar.inc(1); - })?; + let result = cmd.execute( + args.dry_run, + compiler.as_mut().map(|c| c as &mut dyn LuaCompiler), + |dest| { + if let Some(name) = Path::new(dest).file_name() { + bar.set_message(name.to_string_lossy().to_string()); + } + bar.inc(1); + }, + )?; bar.finish_and_clear(); store = result.store; diff --git a/src/luac/mod.rs b/src/luac/mod.rs new file mode 100644 index 0000000..4592ed8 --- /dev/null +++ b/src/luac/mod.rs @@ -0,0 +1,388 @@ +//! Lua bytecode compilation using EdgeTX's `edgetx-luac` WASM module. +//! +//! The module is a compile-only build of EdgeTX's Lua 5.3 fork. Its bytecode matches +//! what the radio expects: `LUA_32BITS` plus EdgeTX's patched header, which dumps +//! `size_t` as an `int` so simulator- and radio-generated `.luac` files are identical. + +use std::io::Read; +use std::path::{Path, PathBuf}; +use thiserror::Error; + +use wamr_rust_sdk::{ + function::Function, instance::Instance, module::Module, runtime::Runtime as WamrRuntime, sys, + value::WasmValue, +}; + +const LUAC_WASM_URL: &str = "https://edgetx-luac.pages.dev/edgetx-luac.wasm"; +const LUAC_WASM_FILE: &str = "edgetx-luac.wasm"; + +/// Interpreter stack size. Lua's parser recurses over nested expressions, so this +/// matches the simulator's stack rather than the smaller SDK default. +const STACK_SIZE: u32 = 256 * 1024; + +/// No app heap: the module carries its own allocator (`wasm_malloc`) in linear memory. +const HEAP_SIZE: u32 = 0; + +// Negative return values of the WASM `compile` export. +const ERR_NO_STATE: i32 = -1; +const ERR_SYNTAX: i32 = -2; +const ERR_DUMP: i32 = -3; + +#[derive(Error, Debug)] +pub enum LuacError { + #[error("cannot determine cache directory")] + CacheDir, + #[error("{context}: {source}")] + Http { + context: String, + source: reqwest::Error, + }, + #[error("{context}: {source}")] + Io { + context: String, + source: std::io::Error, + }, + #[error("downloaded Lua compiler is not a valid WASM binary")] + InvalidWasm, + #[error("Lua compiler: {0}")] + Runtime(String), + #[error("{message}")] + Compile { message: String }, +} + +fn cache_dir() -> Result { + let base = directories::BaseDirs::new().ok_or(LuacError::CacheDir)?; + Ok(base.cache_dir().join("edgetx-cli").join("luac")) +} + +/// Download and cache the EdgeTX Lua compiler WASM module. +/// +/// The upstream artifact carries no version metadata, so a cached copy is reused +/// indefinitely. Delete `/edgetx-cli/luac/edgetx-luac.wasm` to force a refresh. +pub fn ensure_luac_wasm(on_progress: impl Fn(u64, u64)) -> Result { + let dir = cache_dir()?; + let wasm_path = dir.join(LUAC_WASM_FILE); + + if wasm_path.exists() { + if is_valid_wasm(&wasm_path) { + log::debug!("Lua compiler cached at {}", wasm_path.display()); + return Ok(wasm_path); + } + log::debug!("cached file is not valid WASM, re-downloading"); + let _ = std::fs::remove_file(&wasm_path); + } + + log::debug!("downloading {LUAC_WASM_URL}"); + + let resp = reqwest::blocking::get(LUAC_WASM_URL).map_err(|e| LuacError::Http { + context: "downloading the EdgeTX Lua compiler".into(), + source: e, + })?; + if !resp.status().is_success() { + return Err(LuacError::Http { + context: "the EdgeTX Lua compiler is not available".into(), + source: resp.error_for_status().unwrap_err(), + }); + } + + let total = resp.content_length().unwrap_or(0); + + std::fs::create_dir_all(&dir).map_err(|e| LuacError::Io { + context: "creating Lua compiler cache directory".into(), + source: e, + })?; + + let tmp_path = dir.join(format!("{LUAC_WASM_FILE}.tmp")); + let mut file = std::fs::File::create(&tmp_path).map_err(|e| LuacError::Io { + context: format!("creating temp file {}", tmp_path.display()), + source: e, + })?; + + let mut downloaded = 0u64; + let mut reader = resp; + let mut buf = [0u8; 8192]; + loop { + let n = reader.read(&mut buf).map_err(|e| LuacError::Io { + context: "reading Lua compiler download stream".into(), + source: e, + })?; + if n == 0 { + break; + } + std::io::Write::write_all(&mut file, &buf[..n]).map_err(|e| LuacError::Io { + context: "writing Lua compiler to disk".into(), + source: e, + })?; + downloaded += n as u64; + on_progress(downloaded, total); + } + drop(file); + + std::fs::rename(&tmp_path, &wasm_path).map_err(|e| LuacError::Io { + context: format!("renaming temp file to {}", wasm_path.display()), + source: e, + })?; + + if !is_valid_wasm(&wasm_path) { + let _ = std::fs::remove_file(&wasm_path); + return Err(LuacError::InvalidWasm); + } + + Ok(wasm_path) +} + +/// Check if a file starts with the WASM magic bytes (\x00asm). +fn is_valid_wasm(path: &Path) -> bool { + let mut f = match std::fs::File::open(path) { + Ok(f) => f, + Err(_) => return false, + }; + let mut magic = [0u8; 4]; + if f.read_exact(&mut magic).is_err() { + return false; + } + magic == [0x00, b'a', b's', b'm'] +} + +/// Compiles Lua source to EdgeTX bytecode. +/// +/// Exists so package-pipeline code can be exercised without the WASM module. +pub trait LuaCompiler { + fn compile(&mut self, source: &[u8]) -> Result, LuacError>; +} + +/// The `edgetx-luac` WASM module loaded into its own WAMR runtime. +/// +/// WAMR's init/destroy are process-global, so at most one WAMR-backed component +/// (this or `simulator::Runtime`) may be alive at a time. +pub struct Compiler { + /// Holds the WASM instance + module, which borrow `wamr` through transmuted + /// `'static` references (see `Compiler::new`). This field MUST be declared + /// before `wamr`: Rust drops fields in declaration order, so the instance and + /// module must drop *before* the WAMR runtime they point into — otherwise + /// their destructors run against a freed runtime (use-after-free). + state: CompilerState, + #[allow(dead_code)] + wamr: WamrRuntime, + strip: bool, +} + +struct CompilerState { + #[allow(dead_code)] + module: Module<'static>, + instance: Instance<'static>, +} + +impl Compiler { + /// Load the compiler module. `strip` drops debug info from the bytecode. + pub fn new(wasm_bytes: &[u8], strip: bool) -> Result { + let wamr = WamrRuntime::builder() + .use_system_allocator() + .run_as_interpreter() + .build() + .map_err(|e| LuacError::Runtime(format!("creating WAMR runtime: {e}")))?; + + // SAFETY: module and instance live in CompilerState, which is declared + // before `wamr` and therefore dropped before it. + let wamr_ref: &'static WamrRuntime = unsafe { &*(&wamr as *const WamrRuntime) }; + + log::debug!("loading Lua compiler module ({} bytes)", wasm_bytes.len()); + let module = Module::from_vec(wamr_ref, wasm_bytes.to_vec(), "edgetx-luac") + .map_err(|e| LuacError::Runtime(format!("loading WASM module: {e}")))?; + + let module_ref: &'static Module<'static> = unsafe { &*(&module as *const Module<'static>) }; + + // The module is a WASI reactor; WAMR runs its `_initialize` during + // instantiation, so calling it here would re-enter and trap. + let instance = Instance::new_with_args(wamr_ref, module_ref, STACK_SIZE, HEAP_SIZE) + .map_err(|e| LuacError::Runtime(format!("instantiating WASM module: {e}")))?; + + Ok(Self { + state: CompilerState { module, instance }, + wamr, + strip, + }) + } + + /// Download (or reuse) the cached compiler module and load it. + pub fn from_cache_or_download( + strip: bool, + on_progress: impl Fn(u64, u64), + ) -> Result { + let path = ensure_luac_wasm(on_progress)?; + let bytes = std::fs::read(&path).map_err(|e| LuacError::Io { + context: format!("reading {}", path.display()), + source: e, + })?; + Self::new(&bytes, strip) + } + + /// Call a no-argument export returning a pointer into WASM memory, and copy + /// `len` bytes out of it. + fn read_exported_buffer(&self, export: &str, len: usize) -> Result, LuacError> { + let inst = &self.state.instance; + + let func = Function::find_export_func(inst, export) + .map_err(|e| LuacError::Runtime(format!("finding {export}: {e}")))?; + let results = func + .call(inst, &vec![]) + .map_err(|e| LuacError::Runtime(format!("calling {export}: {e}")))?; + + let app_ptr = match results.first() { + Some(WasmValue::I32(v)) => *v as u32, + other => { + return Err(LuacError::Runtime(format!( + "{export} returned unexpected value: {other:?}" + ))); + } + }; + if app_ptr == 0 { + return Err(LuacError::Runtime(format!("{export} returned null"))); + } + + unsafe { + let native = + sys::wasm_runtime_addr_app_to_native(inst.get_inner_instance(), app_ptr as u64); + if native.is_null() { + return Err(LuacError::Runtime(format!( + "{export} pointer is outside WASM memory" + ))); + } + Ok(std::slice::from_raw_parts(native as *const u8, len).to_vec()) + } + } + + /// Read the compiler's error buffer. + fn read_error(&self) -> String { + let inst = &self.state.instance; + + let len = Function::find_export_func(inst, "get_error_len") + .and_then(|f| f.call(inst, &vec![])) + .ok() + .and_then(|r| match r.first() { + Some(WasmValue::I32(v)) if *v > 0 => Some(*v as usize), + _ => None, + }); + + match len { + Some(len) => match self.read_exported_buffer("get_error_ptr", len) { + Ok(bytes) => String::from_utf8_lossy(&bytes).trim().to_string(), + Err(e) => format!("unknown compile error ({e})"), + }, + None => "unknown compile error".to_string(), + } + } +} + +impl LuaCompiler for Compiler { + fn compile(&mut self, source: &[u8]) -> Result, LuacError> { + let inst = &self.state.instance; + + let malloc = Function::find_export_func(inst, "wasm_malloc") + .map_err(|e| LuacError::Runtime(format!("finding wasm_malloc: {e}")))?; + let free = Function::find_export_func(inst, "wasm_free") + .map_err(|e| LuacError::Runtime(format!("finding wasm_free: {e}")))?; + + // Empty sources are valid Lua chunks, but a zero-size allocation is not. + let alloc_len = source.len().max(1); + let results = malloc + .call(inst, &vec![WasmValue::I32(alloc_len as i32)]) + .map_err(|e| LuacError::Runtime(format!("allocating source buffer: {e}")))?; + let ptr = match results.first() { + Some(WasmValue::I32(v)) => *v as u32, + other => { + return Err(LuacError::Runtime(format!( + "wasm_malloc returned unexpected value: {other:?}" + ))); + } + }; + if ptr == 0 { + return Err(LuacError::Runtime("wasm_malloc returned null".into())); + } + + let copied = unsafe { + let native = + sys::wasm_runtime_addr_app_to_native(inst.get_inner_instance(), ptr as u64); + if native.is_null() { + false + } else { + std::ptr::copy_nonoverlapping(source.as_ptr(), native as *mut u8, source.len()); + true + } + }; + if !copied { + let _ = free.call(inst, &vec![WasmValue::I32(ptr as i32)]); + return Err(LuacError::Runtime( + "source buffer is outside WASM memory".into(), + )); + } + + let compile = Function::find_export_func(inst, "compile") + .map_err(|e| LuacError::Runtime(format!("finding compile: {e}")))?; + let call_result = compile.call( + inst, + &vec![ + WasmValue::I32(ptr as i32), + WasmValue::I32(source.len() as i32), + WasmValue::I32(self.strip as i32), + ], + ); + + // The source buffer is no longer needed once compile() returns, on every path. + let _ = free.call(inst, &vec![WasmValue::I32(ptr as i32)]); + + let results = + call_result.map_err(|e| LuacError::Runtime(format!("calling compile: {e}")))?; + let ret = match results.first() { + Some(WasmValue::I32(v)) => *v, + other => { + return Err(LuacError::Runtime(format!( + "compile returned unexpected value: {other:?}" + ))); + } + }; + + match ret { + n if n > 0 => self.read_exported_buffer("get_output_ptr", n as usize), + ERR_SYNTAX => Err(LuacError::Compile { + message: self.read_error(), + }), + ERR_NO_STATE => Err(LuacError::Runtime("could not create a Lua state".into())), + ERR_DUMP => Err(LuacError::Runtime("writing bytecode failed".into())), + n => Err(LuacError::Runtime(format!("compile returned {n}"))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn valid_wasm_detects_magic() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("mod.wasm"); + + std::fs::write(&path, b"\0asm\x01\0\0\0").unwrap(); + assert!(is_valid_wasm(&path)); + + std::fs::write(&path, b"").unwrap(); + assert!(!is_valid_wasm(&path)); + + std::fs::write(&path, b"ab").unwrap(); + assert!(!is_valid_wasm(&path)); + } + + #[test] + fn valid_wasm_rejects_missing_file() { + let dir = TempDir::new().unwrap(); + assert!(!is_valid_wasm(&dir.path().join("nope.wasm"))); + } + + #[test] + fn cache_dir_ends_with_luac() { + let dir = cache_dir().unwrap(); + assert!(dir.ends_with("edgetx-cli/luac"), "got {}", dir.display()); + } +} diff --git a/src/main.rs b/src/main.rs index f61f087..9ca7308 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ mod cli; mod device; +mod luac; mod manifest; mod packages; mod radio; diff --git a/src/manifest/mod.rs b/src/manifest/mod.rs index 6f0c27d..c2964a0 100644 --- a/src/manifest/mod.rs +++ b/src/manifest/mod.rs @@ -739,6 +739,22 @@ impl Manifest { &self.files, ] } + + /// Returns every content group for in-place editing, libraries first. + pub(crate) fn all_groups_mut(&mut self) -> Vec<&mut Vec> { + vec![ + &mut self.libraries, + &mut self.tools, + &mut self.telemetry, + &mut self.functions, + &mut self.mixes, + &mut self.widgets, + &mut self.sounds, + &mut self.images, + &mut self.themes, + &mut self.files, + ] + } } #[cfg(test)] diff --git a/src/packages/build.rs b/src/packages/build.rs new file mode 100644 index 0000000..000f052 --- /dev/null +++ b/src/packages/build.rs @@ -0,0 +1,583 @@ +use std::path::Path; + +use crate::luac::LuaCompiler; +use crate::manifest::{ContentItem, Manifest}; +use crate::packages::path::PackagePath; +use crate::radio; + +use super::PackageError; +use super::transfer::{build_exclude, copy_content_items, count_files, count_lua_files}; + +/// Layout of a built package. +pub enum BuildLayout { + /// `edgetx.yml` at the root with content at each item's `path`, ready for + /// `pkg install ./`. + Package, + /// Content at each item's install destination and no manifest, so the tree + /// extracts straight onto the SD card root. + SdOverlay, +} + +pub struct BuildOptions<'a> { + pub manifest: &'a Manifest, + pub manifest_dir: &'a Path, + pub out_dir: &'a Path, + pub include_dev: bool, + pub layout: BuildLayout, +} + +#[derive(Debug)] +pub struct BuildResult { + /// The manifest written to the output directory. `None` for [`BuildLayout::SdOverlay`]. + pub manifest: Option, + pub files_copied: usize, + pub scripts_compiled: usize, +} + +/// Copy a package into `out_dir` and compile a `.luac` next to every `.lua`. +/// +/// Both the sources and the generated bytecode land in the output, because the +/// radio only discovers some script types (widgets in particular) by their `.lua`. +/// Everything happens on local disk: SD cards are far too slow to compile against. +pub fn build_package( + opts: &BuildOptions, + compiler: &mut dyn LuaCompiler, + on_copy: &mut dyn FnMut(&str), + on_compile: &mut dyn FnMut(&str), +) -> Result { + let staged = match opts.layout { + BuildLayout::SdOverlay => { + copy_content_items( + opts.manifest, + opts.manifest_dir, + opts.out_dir, + opts.include_dev, + on_copy, + )? + .1 + } + BuildLayout::Package => stage_sources(opts, on_copy)?, + }; + + let scripts_compiled = compile_staged(opts.out_dir, &staged, compiler, on_compile)?; + + let manifest = match opts.layout { + BuildLayout::SdOverlay => None, + BuildLayout::Package => { + let m = built_manifest(opts); + write_manifest(&m, opts.out_dir)?; + Some(m) + } + }; + + Ok(BuildResult { + manifest, + files_copied: staged.len(), + scripts_compiled, + }) +} + +/// Copy content to `out_dir` at each item's own `path`, so the emitted manifest +/// resolves against the output root and `dest` still applies at install time. +fn stage_sources( + opts: &BuildOptions, + on_copy: &mut dyn FnMut(&str), +) -> Result, PackageError> { + let mut staged = Vec::new(); + + for item in opts.manifest.content_items(opts.include_dev) { + let source_root = opts + .manifest + .resolve_content_path(opts.manifest_dir, &item.path) + .map_err(|e| PackageError::ContentResolve { + path: item.path.clone(), + source: e, + })?; + + let exclude = build_exclude(opts.manifest.package.binary, &item); + let copy_opts = radio::copy::CopyOptions { + dry_run: false, + exclude: &exclude, + }; + + let mut on = |dest: &Path| { + if let Ok(rel) = dest.strip_prefix(opts.out_dir) { + staged.push(PackagePath::new(rel.to_string_lossy())); + } + on_copy(&dest.display().to_string()); + }; + + radio::copy::copy_paths( + &source_root, + opts.out_dir, + &[radio::copy::CopyPath::same(item.path.as_str())], + ©_opts, + &mut on, + )?; + } + + Ok(staged) +} + +/// Compile every staged `.lua`, writing the bytecode beside it. +fn compile_staged( + out_dir: &Path, + staged: &[PackagePath], + compiler: &mut dyn LuaCompiler, + on_compile: &mut dyn FnMut(&str), +) -> Result { + let mut compiled = 0; + + for rel in staged { + let Some(compiled_rel) = rel.compiled_path() else { + continue; + }; + + let dest = out_dir.join(compiled_rel.as_str()); + if dest.exists() { + log::info!("{compiled_rel} ships with the package, not recompiling"); + continue; + } + + let src = out_dir.join(rel.as_str()); + let source = std::fs::read(&src).map_err(|e| PackageError::Io { + context: format!("reading {}", src.display()), + source: e, + })?; + + on_compile(rel.as_str()); + + let bytecode = compiler + .compile(&source) + .map_err(|e| PackageError::Compile { + path: rel.to_string(), + source: e, + })?; + + std::fs::write(&dest, &bytecode).map_err(|e| PackageError::Io { + context: format!("writing {}", dest.display()), + source: e, + })?; + + copy_mtime(&src, &dest); + compiled += 1; + } + + Ok(compiled) +} + +/// Give the bytecode its source's timestamp, as the radio does when it compiles a +/// script itself. The radio loads whichever of the two is newer and prefers the +/// bytecode on a tie, so matching timestamps stop it from recompiling. +pub(crate) fn copy_mtime(src: &Path, dest: &Path) { + let result = std::fs::metadata(src) + .and_then(|m| m.modified()) + .and_then(|t| { + std::fs::File::options() + .write(true) + .open(dest) + .and_then(|f| f.set_modified(t)) + }); + + if let Err(e) = result { + log::warn!("could not set the timestamp on {}: {e}", dest.display()); + } +} + +/// Rewrite the manifest to describe the built tree rather than the source tree. +fn built_manifest(opts: &BuildOptions) -> Manifest { + let mut m = opts.manifest.clone(); + + // Content now sits at the output root, and neither screenshots nor variant + // manifests are part of a build. + m.package.source_dir = Default::default(); + m.package.screenshots.clear(); + m.package.variants.clear(); + // Bytecode ships with the package, so `.luac` must survive the install copy. + m.package.binary = true; + + let include_dev = opts.include_dev; + let out_dir = opts.out_dir; + + for group in m.all_groups_mut() { + if !include_dev { + group.retain(|item| !item.dev); + } + + // Items naming a single `.lua` file need a second entry for its bytecode; + // items naming a directory pick it up on their own. + let compiled: Vec = group + .iter() + .filter_map(|item| { + let path = item.path.compiled_path()?; + if !out_dir.join(path.as_str()).exists() { + return None; + } + let dest = match &item.dest { + Some(d) => match d.compiled_path() { + Some(c) => Some(c), + None => { + log::warn!( + "no bytecode destination for {:?}, shipping sources only", + item.name + ); + return None; + } + }, + None => None, + }; + Some(ContentItem { + name: format!("{} (compiled)", item.name), + path, + dest, + depends: item.depends.clone(), + exclude: Vec::new(), + dev: item.dev, + }) + }) + .collect(); + + group.extend(compiled); + } + + m +} + +fn write_manifest(m: &Manifest, out_dir: &Path) -> Result<(), PackageError> { + let path = out_dir.join(crate::manifest::FILE_NAME); + let yaml = serde_yml::to_string(m).map_err(|e| PackageError::Io { + context: format!("serializing {}", path.display()), + source: std::io::Error::other(e), + })?; + std::fs::write(&path, yaml).map_err(|e| PackageError::Io { + context: format!("writing {}", path.display()), + source: e, + }) +} + +/// Turn a package id into a filename: `Jurg/MyTool` -> `Jurg-MyTool`. +pub fn archive_base_name(id: &str) -> String { + id.replace(['/', '\\'], "-") +} + +/// Progress steps a build reports: one per file copied, one per script compiled. +pub fn count_steps(manifest: &Manifest, manifest_dir: &Path, include_dev: bool) -> usize { + count_files(manifest_dir, manifest, include_dev) + + count_lua_files(manifest_dir, manifest, include_dev) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::luac::LuacError; + use crate::manifest; + use std::fs; + use tempfile::TempDir; + + /// Stand-in for the WASM compiler: prefixes the source so tests can tell + /// bytecode from source, and can be told to fail. + struct FakeCompiler { + fail: bool, + calls: Vec, + } + + impl FakeCompiler { + fn new() -> Self { + Self { + fail: false, + calls: Vec::new(), + } + } + + fn failing() -> Self { + Self { + fail: true, + calls: Vec::new(), + } + } + } + + impl LuaCompiler for FakeCompiler { + fn compile(&mut self, source: &[u8]) -> Result, LuacError> { + self.calls.push(String::from_utf8_lossy(source).to_string()); + if self.fail { + return Err(LuacError::Compile { + message: "input:1: syntax error near 'oops'".into(), + }); + } + let mut out = b"\x1bLuaS".to_vec(); + out.extend_from_slice(source); + Ok(out) + } + } + + /// Create a source package: `edgetx.yml` plus each (path, contents) file. + fn source_package(yml: &str, files: &[(&str, &str)]) -> TempDir { + let dir = TempDir::new().unwrap(); + fs::write(dir.path().join(manifest::FILE_NAME), yml).unwrap(); + for (path, contents) in files { + let p = dir.path().join(path); + fs::create_dir_all(p.parent().unwrap()).unwrap(); + fs::write(&p, contents).unwrap(); + } + dir + } + + const TOOL_YML: &str = r#" +package: + id: github.com/Jurg/MyTool + name: My Tool + description: A test tool +tools: + - name: MyTool + path: SCRIPTS/TOOLS/MyTool +"#; + + fn build( + src: &TempDir, + out: &TempDir, + layout: BuildLayout, + compiler: &mut dyn LuaCompiler, + ) -> Result { + let m = manifest::load(src.path()).unwrap(); + build_package( + &BuildOptions { + manifest: &m, + manifest_dir: src.path(), + out_dir: out.path(), + include_dev: false, + layout, + }, + compiler, + &mut |_| {}, + &mut |_| {}, + ) + } + + #[test] + fn package_layout_compiles_next_to_sources() { + let src = source_package( + TOOL_YML, + &[ + ("SCRIPTS/TOOLS/MyTool/main.lua", "return 1"), + ("SCRIPTS/TOOLS/MyTool/lib.lua", "return 2"), + ("SCRIPTS/TOOLS/MyTool/icon.png", "png"), + ], + ); + let out = TempDir::new().unwrap(); + + let mut compiler = FakeCompiler::new(); + let result = build(&src, &out, BuildLayout::Package, &mut compiler).unwrap(); + + assert_eq!(result.scripts_compiled, 2); + for name in ["main.lua", "main.luac", "lib.lua", "lib.luac", "icon.png"] { + assert!( + out.path().join("SCRIPTS/TOOLS/MyTool").join(name).exists(), + "{name} missing from the build" + ); + } + assert_eq!( + fs::read(out.path().join("SCRIPTS/TOOLS/MyTool/main.luac")).unwrap(), + b"\x1bLuaSreturn 1" + ); + } + + #[test] + fn package_layout_emits_installable_manifest() { + let src = source_package(TOOL_YML, &[("SCRIPTS/TOOLS/MyTool/main.lua", "return 1")]); + let out = TempDir::new().unwrap(); + + let mut compiler = FakeCompiler::new(); + let result = build(&src, &out, BuildLayout::Package, &mut compiler).unwrap(); + + let emitted = result.manifest.unwrap(); + assert!(emitted.package.binary, "bytecode must survive the install"); + assert!(emitted.package.source_dir.0.is_empty()); + + // The output directory loads and validates as a package of its own. + let reloaded = manifest::load(out.path()).unwrap(); + reloaded.validate(out.path()).unwrap(); + assert_eq!(reloaded.package.id, "github.com/Jurg/MyTool"); + } + + #[test] + fn package_layout_drops_dev_items() { + let yml = r#" +package: + id: github.com/Jurg/MyTool + description: A test tool +tools: + - name: MyTool + path: SCRIPTS/TOOLS/MyTool + - name: Fixtures + path: SCRIPTS/TOOLS/Fixtures + dev: true +"#; + let src = source_package( + yml, + &[ + ("SCRIPTS/TOOLS/MyTool/main.lua", "return 1"), + ("SCRIPTS/TOOLS/Fixtures/fake.lua", "return 2"), + ], + ); + let out = TempDir::new().unwrap(); + + let mut compiler = FakeCompiler::new(); + let result = build(&src, &out, BuildLayout::Package, &mut compiler).unwrap(); + + assert!(!out.path().join("SCRIPTS/TOOLS/Fixtures").exists()); + assert_eq!(result.manifest.unwrap().tools.len(), 1); + // The dev script must not have been compiled either. + assert_eq!(compiler.calls, vec!["return 1"]); + } + + #[test] + fn single_file_items_get_a_bytecode_entry() { + let yml = r#" +package: + id: github.com/Jurg/MyTool + description: A test tool +functions: + - name: Logger + path: SCRIPTS/FUNCTIONS/logger.lua +"#; + let src = source_package(yml, &[("SCRIPTS/FUNCTIONS/logger.lua", "return 1")]); + let out = TempDir::new().unwrap(); + + let mut compiler = FakeCompiler::new(); + let result = build(&src, &out, BuildLayout::Package, &mut compiler).unwrap(); + + let emitted = result.manifest.unwrap(); + let paths: Vec<&str> = emitted.functions.iter().map(|i| i.path.as_str()).collect(); + assert_eq!( + paths, + vec![ + "SCRIPTS/FUNCTIONS/logger.lua", + "SCRIPTS/FUNCTIONS/logger.luac" + ], + "the generated bytecode needs its own entry or the install would drop it" + ); + } + + #[test] + fn stale_bytecode_is_replaced() { + let src = source_package( + TOOL_YML, + &[ + ("SCRIPTS/TOOLS/MyTool/main.lua", "return 1"), + ("SCRIPTS/TOOLS/MyTool/main.luac", "stale bytecode"), + ], + ); + let out = TempDir::new().unwrap(); + + let mut compiler = FakeCompiler::new(); + build(&src, &out, BuildLayout::Package, &mut compiler).unwrap(); + + assert_eq!( + fs::read(out.path().join("SCRIPTS/TOOLS/MyTool/main.luac")).unwrap(), + b"\x1bLuaSreturn 1", + "a stale .luac in the sources must not reach the build" + ); + } + + #[test] + fn bundled_bytecode_is_kept() { + let yml = r#" +package: + id: github.com/Jurg/MyTool + description: A test tool + binary: true +tools: + - name: MyTool + path: SCRIPTS/TOOLS/MyTool +"#; + let src = source_package( + yml, + &[ + ("SCRIPTS/TOOLS/MyTool/main.lua", "return 1"), + ("SCRIPTS/TOOLS/MyTool/main.luac", "shipped bytecode"), + ], + ); + let out = TempDir::new().unwrap(); + + let mut compiler = FakeCompiler::new(); + let result = build(&src, &out, BuildLayout::Package, &mut compiler).unwrap(); + + assert_eq!(result.scripts_compiled, 0); + assert!(compiler.calls.is_empty()); + assert_eq!( + fs::read(out.path().join("SCRIPTS/TOOLS/MyTool/main.luac")).unwrap(), + b"shipped bytecode" + ); + } + + #[test] + fn bytecode_matches_the_source_timestamp() { + let src = source_package(TOOL_YML, &[("SCRIPTS/TOOLS/MyTool/main.lua", "return 1")]); + let out = TempDir::new().unwrap(); + + let mut compiler = FakeCompiler::new(); + build(&src, &out, BuildLayout::Package, &mut compiler).unwrap(); + + let dir = out.path().join("SCRIPTS/TOOLS/MyTool"); + let lua = fs::metadata(dir.join("main.lua")) + .unwrap() + .modified() + .unwrap(); + let luac = fs::metadata(dir.join("main.luac")) + .unwrap() + .modified() + .unwrap(); + assert_eq!(lua, luac); + } + + #[test] + fn compile_errors_name_the_script() { + let src = source_package(TOOL_YML, &[("SCRIPTS/TOOLS/MyTool/main.lua", "return 1")]); + let out = TempDir::new().unwrap(); + + let mut compiler = FakeCompiler::failing(); + let err = build(&src, &out, BuildLayout::Package, &mut compiler).unwrap_err(); + + let msg = err.to_string(); + assert!(msg.contains("SCRIPTS/TOOLS/MyTool/main.lua"), "got {msg}"); + assert!(msg.contains("syntax error"), "got {msg}"); + } + + #[test] + fn sd_overlay_layout_uses_install_destinations() { + let yml = r#" +package: + id: github.com/Jurg/MyTool + description: A test tool +tools: + - name: MyTool + path: src/MyTool + dest: SCRIPTS/TOOLS/MyTool +"#; + let src = source_package(yml, &[("src/MyTool/main.lua", "return 1")]); + let out = TempDir::new().unwrap(); + + let mut compiler = FakeCompiler::new(); + let result = build(&src, &out, BuildLayout::SdOverlay, &mut compiler).unwrap(); + + assert!(out.path().join("SCRIPTS/TOOLS/MyTool/main.lua").exists()); + assert!(out.path().join("SCRIPTS/TOOLS/MyTool/main.luac").exists()); + assert!(!out.path().join("src").exists(), "sources must be remapped"); + assert!( + !out.path().join(manifest::FILE_NAME).exists(), + "an SD overlay carries no manifest" + ); + assert!(result.manifest.is_none()); + } + + #[test] + fn archive_base_name_is_filename_safe() { + assert_eq!(archive_base_name("Jurg/MyTool"), "Jurg-MyTool"); + assert_eq!( + archive_base_name("github.com/Jurg/MyTool"), + "github.com-Jurg-MyTool" + ); + assert_eq!(archive_base_name("MyTool"), "MyTool"); + } +} diff --git a/src/packages/install.rs b/src/packages/install.rs index e46e2f8..5fd7154 100644 --- a/src/packages/install.rs +++ b/src/packages/install.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use crate::luac::LuaCompiler; use crate::manifest::{self, Manifest, RadioCapabilities}; use crate::packages::path::PackagePath; use crate::radio; @@ -9,7 +10,7 @@ use crate::source::{PackageRef, resolve}; use super::PackageError; use super::file_list::PackageFileList; use super::store::{InstalledPackage, PackageStore}; -use super::transfer::{copy_content_items, count_files}; +use super::transfer::{count_files, count_lua_files, stage_and_copy}; /// InstallOptions configures an install operation. pub struct InstallOptions { @@ -160,15 +161,25 @@ impl InstallCommand { }) } - /// Returns the number of files that will be copied. - pub fn total_files(&self) -> usize { - count_files(&self.manifest_dir, &self.manifest, self.include_dev) + /// Returns the number of progress steps the install will report. Pre-compiling + /// adds two per script: one to compile it, one to copy the bytecode across. + pub fn total_files(&self, pre_compile: bool) -> usize { + let files = count_files(&self.manifest_dir, &self.manifest, self.include_dev); + if pre_compile { + files + 2 * count_lua_files(&self.manifest_dir, &self.manifest, self.include_dev) + } else { + files + } } /// Execute copies the files and updates the state. + /// + /// With a compiler the package is staged and compiled locally first, so a + /// broken script fails the install before the SD card or the store is touched. pub fn execute( self, dry_run: bool, + compiler: Option<&mut dyn LuaCompiler>, mut on_file: impl FnMut(&str), ) -> Result { let mut total_copied = 0; @@ -180,11 +191,12 @@ impl InstallCommand { store.remove(id); } - let (n, mut copied_files) = copy_content_items( + let (n, mut copied_files) = stage_and_copy( &self.manifest, &self.manifest_dir, &sd_root, self.include_dev, + compiler, &mut on_file, )?; total_copied = n; @@ -258,7 +270,7 @@ tools: assert_eq!(cmd.package.id, "example.com/test/test-pkg"); assert_eq!(cmd.package.channel, Channel::Local); - let result = cmd.execute(false, |_| {}).unwrap(); + let result = cmd.execute(false, None, |_| {}).unwrap(); assert_eq!(result.files_copied, 1); // Verify state @@ -295,7 +307,7 @@ tools: }) .unwrap(); - cmd.execute(false, |_| {}).unwrap(); + cmd.execute(false, None, |_| {}).unwrap(); let store = PackageStore::load(sd_dir.path().to_path_buf()).unwrap(); let file_list = PackageFileList::load(&store.file_list_dir, "example.com/test/test-pkg"); @@ -338,7 +350,7 @@ tools: }) .unwrap(); - let result = cmd.execute(true, |_| {}).unwrap(); + let result = cmd.execute(true, None, |_| {}).unwrap(); assert_eq!(result.files_copied, 0); // Verify no state was saved @@ -371,7 +383,7 @@ tools: radio: None, }) .unwrap(); - cmd.execute(false, |_| {}).unwrap(); + cmd.execute(false, None, |_| {}).unwrap(); // Reinstall same source should succeed let cmd = InstallCommand::new(InstallOptions { @@ -384,7 +396,7 @@ tools: radio: None, }) .unwrap(); - let result = cmd.execute(false, |_| {}).unwrap(); + let result = cmd.execute(false, None, |_| {}).unwrap(); assert_eq!(result.files_copied, 1); let store = PackageStore::load(sd_dir.path().to_path_buf()).unwrap(); @@ -416,7 +428,7 @@ tools: radio: None, }) .unwrap(); - cmd.execute(false, |_| {}).unwrap(); + cmd.execute(false, None, |_| {}).unwrap(); assert!(sd_dir.path().join("SCRIPTS/TOOLS/MyTool/main.lua").exists()); // Call new() but don't execute — files should still be on disk @@ -460,7 +472,7 @@ tools: radio: None, }) .unwrap(); - cmd.execute(false, |_| {}).unwrap(); + cmd.execute(false, None, |_| {}).unwrap(); // Package B: different id, overlapping path → should conflict let (pkg_dir_b, _) = setup_local_package( @@ -512,7 +524,7 @@ tools: radio: None, }) .unwrap(); - cmd.execute(false, |_| {}).unwrap(); + cmd.execute(false, None, |_| {}).unwrap(); // Same id from different dir — should succeed (replaces) let (pkg_dir_b, _) = setup_local_package( @@ -564,7 +576,7 @@ tools: radio: None, }) .unwrap(); - cmd.execute(false, |_| {}).unwrap(); + cmd.execute(false, None, |_| {}).unwrap(); // Package B: same name, different source, different path — no conflict let (pkg_dir_b, _) = setup_local_package( @@ -617,7 +629,7 @@ tools: radio: None, }) .unwrap(); - cmd.execute(false, |_| {}).unwrap(); + cmd.execute(false, None, |_| {}).unwrap(); let store = PackageStore::load(sd_dir.path().to_path_buf()).unwrap(); let file_list = PackageFileList::load(&store.file_list_dir, "example.com/test/bare-tool"); @@ -748,7 +760,7 @@ widgets: }) .unwrap(); - let result = cmd.execute(false, |_| {}).unwrap(); + let result = cmd.execute(false, None, |_| {}).unwrap(); assert!(result.files_copied > 0); // Color variant's widget should be installed @@ -790,7 +802,7 @@ widgets: }) .unwrap(); - let result = cmd.execute(false, |_| {}).unwrap(); + let result = cmd.execute(false, None, |_| {}).unwrap(); assert!(result.files_copied > 0); // Generic BW variant should be selected (212x64 doesn't match 128x64 specific) @@ -831,7 +843,7 @@ widgets: }) .unwrap(); - let result = cmd.execute(false, |_| {}).unwrap(); + let result = cmd.execute(false, None, |_| {}).unwrap(); assert!(result.files_copied > 0); // Specific 128x64 variant should win over generic BW @@ -928,4 +940,136 @@ widgets: let err = result.err().expect("should fail"); assert!(err.to_string().contains("no matching variant")); } + + /// Stand-in for the WASM compiler, so these tests need no network. + struct FakeCompiler { + fail: bool, + } + + impl crate::luac::LuaCompiler for FakeCompiler { + fn compile(&mut self, source: &[u8]) -> Result, crate::luac::LuacError> { + if self.fail { + return Err(crate::luac::LuacError::Compile { + message: "input:1: syntax error near 'oops'".into(), + }); + } + let mut out = b"\x1bLuaS".to_vec(); + out.extend_from_slice(source); + Ok(out) + } + } + + const PRE_COMPILE_YML: &str = r#" +package: + id: example.com/test/test-pkg + description: "Test" +tools: + - name: MyTool + path: SCRIPTS/TOOLS/MyTool +"#; + + #[test] + fn test_install_pre_compile_writes_bytecode() { + let (pkg_dir, sd_dir) = setup_local_package( + PRE_COMPILE_YML, + &[ + "SCRIPTS/TOOLS/MyTool/main.lua", + "SCRIPTS/TOOLS/MyTool/x.png", + ], + ); + + let cmd = InstallCommand::new(InstallOptions { + sd_root: sd_dir.path().to_path_buf(), + pkg_ref: PackageRef::Local { + path: pkg_dir.path().to_path_buf(), + variant: String::new(), + }, + dev: false, + radio: None, + }) + .unwrap(); + + // 2 files copied + 1 script compiled + 1 bytecode copied. + assert_eq!(cmd.total_files(true), 4); + + let mut compiler = FakeCompiler { fail: false }; + let result = cmd.execute(false, Some(&mut compiler), |_| {}).unwrap(); + assert_eq!(result.files_copied, 3); + + let tool = sd_dir.path().join("SCRIPTS/TOOLS/MyTool"); + assert!(tool.join("main.lua").exists()); + assert!(tool.join("main.luac").exists()); + + // The bytecode must be tracked, or removing the package would orphan it. + let store = PackageStore::load(sd_dir.path().to_path_buf()).unwrap(); + let list = PackageFileList::load(&store.file_list_dir, "example.com/test/test-pkg"); + assert!( + list.files() + .iter() + .any(|f| f == "SCRIPTS/TOOLS/MyTool/main.luac"), + "bytecode missing from {:?}", + list.files() + ); + } + + #[test] + fn test_install_pre_compile_failure_leaves_nothing_behind() { + let (pkg_dir, sd_dir) = + setup_local_package(PRE_COMPILE_YML, &["SCRIPTS/TOOLS/MyTool/main.lua"]); + + let cmd = InstallCommand::new(InstallOptions { + sd_root: sd_dir.path().to_path_buf(), + pkg_ref: PackageRef::Local { + path: pkg_dir.path().to_path_buf(), + variant: String::new(), + }, + dev: false, + radio: None, + }) + .unwrap(); + + let mut compiler = FakeCompiler { fail: true }; + let err = match cmd.execute(false, Some(&mut compiler), |_| {}) { + Err(e) => e, + Ok(_) => panic!("a broken script must fail the install"), + }; + assert!(err.to_string().contains("main.lua"), "got {err}"); + + // Compilation happens in staging, before anything reaches the card. + assert!(!sd_dir.path().join("SCRIPTS").exists()); + let store = PackageStore::load(sd_dir.path().to_path_buf()).unwrap(); + assert!(store.packages().is_empty()); + } + + #[test] + fn test_install_pre_compile_keeps_bytecode_current() { + let (pkg_dir, sd_dir) = + setup_local_package(PRE_COMPILE_YML, &["SCRIPTS/TOOLS/MyTool/main.lua"]); + + let cmd = InstallCommand::new(InstallOptions { + sd_root: sd_dir.path().to_path_buf(), + pkg_ref: PackageRef::Local { + path: pkg_dir.path().to_path_buf(), + variant: String::new(), + }, + dev: false, + radio: None, + }) + .unwrap(); + + let mut compiler = FakeCompiler { fail: false }; + cmd.execute(false, Some(&mut compiler), |_| {}).unwrap(); + + // The radio reloads the source whenever it looks newer than the bytecode. + let tool = sd_dir.path().join("SCRIPTS/TOOLS/MyTool"); + let lua = std::fs::metadata(tool.join("main.lua")) + .unwrap() + .modified() + .unwrap(); + let luac = std::fs::metadata(tool.join("main.luac")) + .unwrap() + .modified() + .unwrap(); + assert!(luac >= lua, "bytecode looks stale next to its source"); + } } diff --git a/src/packages/mod.rs b/src/packages/mod.rs index 9c6afe1..31245b3 100644 --- a/src/packages/mod.rs +++ b/src/packages/mod.rs @@ -1,3 +1,4 @@ +pub mod build; pub mod file_list; pub mod info; pub mod install; @@ -35,6 +36,16 @@ pub enum PackageError { path: PackagePath, source: ManifestError, }, + #[error("compiling {path}: {source}")] + Compile { + path: String, + source: crate::luac::LuacError, + }, + #[error("{context}: {source}")] + Io { + context: String, + source: std::io::Error, + }, #[error(transparent)] Store(#[from] StoreError), #[error(transparent)] diff --git a/src/packages/remove.rs b/src/packages/remove.rs index 163acd0..cd22618 100644 --- a/src/packages/remove.rs +++ b/src/packages/remove.rs @@ -99,11 +99,13 @@ impl RemoveCommand { } } - // Find compiled .luac companions that exist on disk + // Find compiled .luac companions that exist on disk. Pre-compiled installs + // already track their bytecode, so skip anything the list covers. let luac_files: Vec = files .iter() .filter(|f| f.ends_with(".lua")) .map(|f| format!("{f}c")) + .filter(|luac| !files.contains(luac)) .filter(|luac| store.sd_root().join(luac).exists()) .collect(); @@ -168,6 +170,51 @@ mod tests { (sd_dir, TEST_ID.into()) } + /// A pre-compiled install tracks its bytecode explicitly, so the derived + /// companion must not count it a second time. + fn setup_precompiled_package() -> (TempDir, String) { + let (sd_dir, source) = setup_installed_package(); + let store = PackageStore::load(sd_dir.path().to_path_buf()).unwrap(); + + PackageFileList::new( + TEST_ID.into(), + vec![ + "SCRIPTS/TOOLS/MyTool/main.lua".into(), + "SCRIPTS/TOOLS/MyTool/main.luac".into(), + "SCRIPTS/TOOLS/MyTool/".into(), + ], + ) + .save(&store.file_list_dir) + .unwrap(); + + (sd_dir, source) + } + + #[test] + fn test_remove_tracked_luac_counted_once() { + let (sd_dir, source) = setup_precompiled_package(); + let sd = sd_dir.path(); + + let cmd = RemoveCommand::new(RemoveOptions { + sd_root: sd.to_path_buf(), + query: source, + }) + .unwrap(); + + assert_eq!(cmd.files.len(), 2); + assert!( + cmd.luac_files.is_empty(), + "tracked bytecode must not be listed twice: {:?}", + cmd.luac_files + ); + assert_eq!(cmd.total_files(), 2); + + let result = cmd.execute(false, |_| {}).unwrap(); + assert_eq!(result.files_removed, 2); + assert!(!sd.join("SCRIPTS/TOOLS/MyTool/main.lua").exists()); + assert!(!sd.join("SCRIPTS/TOOLS/MyTool/main.luac").exists()); + } + #[test] fn test_remove_package_with_luac() { let (sd_dir, source) = setup_installed_package(); diff --git a/src/packages/transfer.rs b/src/packages/transfer.rs index 782b7c5..2030395 100644 --- a/src/packages/transfer.rs +++ b/src/packages/transfer.rs @@ -1,10 +1,81 @@ use std::path::Path; +use crate::luac::LuaCompiler; use crate::manifest::{ContentItem, Manifest}; use crate::packages::path::PackagePath; use crate::radio; use super::PackageError; +use super::build::{BuildLayout, BuildOptions, build_package, copy_mtime}; + +/// Copy a package to the SD card, optionally pre-compiling its Lua scripts. +/// +/// With a compiler the package is first built into a local staging directory -- +/// compiling against an SD card would be far too slow -- and the finished tree, +/// sources and bytecode alike, is copied across in a single pass. +pub(crate) fn stage_and_copy( + manifest: &Manifest, + manifest_dir: &Path, + sd_root: &Path, + include_dev: bool, + compiler: Option<&mut dyn LuaCompiler>, + on_file: &mut dyn FnMut(&str), +) -> Result<(usize, Vec), PackageError> { + let Some(compiler) = compiler else { + return copy_content_items(manifest, manifest_dir, sd_root, include_dev, on_file); + }; + + let staging = tempfile::TempDir::new().map_err(|e| PackageError::Io { + context: "creating a staging directory".into(), + source: e, + })?; + + let built = build_package( + &BuildOptions { + manifest, + manifest_dir, + out_dir: staging.path(), + include_dev, + layout: BuildLayout::Package, + }, + compiler, + // Staging is local and fast, so only the compile step reports progress. + &mut |_| {}, + &mut *on_file, + )?; + + // The built manifest is marked binary, so the bytecode survives the copy. + let staged = built + .manifest + .expect("the package layout always emits a manifest"); + + let (copied, files) = + copy_content_items(&staged, staging.path(), sd_root, include_dev, on_file)?; + fix_luac_mtimes(sd_root, &files); + + Ok((copied, files)) +} + +/// Give each copied `.luac` the timestamp of its source on the SD card. +/// +/// `std::fs::copy` does not preserve timestamps and the copy order is arbitrary, +/// so without this a `.lua` can end up looking newer than its bytecode -- which +/// is exactly when the radio throws the bytecode away and recompiles. +fn fix_luac_mtimes(sd_root: &Path, files: &[PackagePath]) { + let all: std::collections::HashSet<&str> = files.iter().map(|f| f.as_str()).collect(); + + for file in files { + let name = file.as_str(); + if !name.ends_with(".luac") { + continue; + } + // "main.luac" -> "main.lua" + let Some(source) = name.strip_suffix('c').filter(|s| all.contains(s)) else { + continue; + }; + copy_mtime(&sd_root.join(source), &sd_root.join(name)); + } +} /// Copy all content items from a manifest to the SD card root. /// @@ -15,7 +86,7 @@ pub(crate) fn copy_content_items( manifest_dir: &Path, sd_root: &Path, include_dev: bool, - on_file: &mut impl FnMut(&str), + on_file: &mut dyn FnMut(&str), ) -> Result<(usize, Vec), PackageError> { let mut total_copied = 0; let mut copied_files = Vec::new(); @@ -68,8 +139,34 @@ pub(crate) fn count_files(manifest_dir: &Path, m: &Manifest, include_dev: bool) total } +/// Count the Lua sources a pre-compile pass would compile. +pub(crate) fn count_lua_files(manifest_dir: &Path, m: &Manifest, include_dev: bool) -> usize { + let mut total = 0; + for item in m.content_items(include_dev) { + let Ok(source_root) = m.resolve_content_path(manifest_dir, &item.path) else { + continue; + }; + let exclude = build_exclude(m.package.binary, &item); + + // WalkDir yields a single-file root as its only entry, so this covers + // both directory and single-file content items. + for entry in walkdir::WalkDir::new(source_root.join(item.path.as_str())) + .into_iter() + .flatten() + { + if entry.file_type().is_file() + && entry.path().extension().is_some_and(|e| e == "lua") + && !radio::copy::is_excluded(entry.path(), &exclude) + { + total += 1; + } + } + } + total +} + /// Build exclude patterns for a content item. -fn build_exclude(binary: bool, item: &ContentItem) -> Vec { +pub(crate) fn build_exclude(binary: bool, item: &ContentItem) -> Vec { if binary { item.exclude.clone() } else { diff --git a/src/packages/update.rs b/src/packages/update.rs index 3f25c90..e474d8b 100644 --- a/src/packages/update.rs +++ b/src/packages/update.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use crate::luac::LuaCompiler; use crate::manifest::{self, Manifest}; use crate::source::version::Channel; use crate::source::{PackageRef, resolve}; @@ -7,7 +8,7 @@ use crate::source::{PackageRef, resolve}; use super::PackageError; use super::file_list::PackageFileList; use super::store::{InstalledPackage, PackageStore}; -use super::transfer::{copy_content_items, count_files}; +use super::transfer::{count_files, count_lua_files, stage_and_copy}; /// UpdateOptions configures the construction of an UpdateCommand. pub struct UpdateOptions<'a> { @@ -155,15 +156,25 @@ impl UpdateCommand { }) } - /// Returns the number of files that will be copied. - pub fn total_files(&self) -> usize { - count_files(&self.manifest_dir, &self.manifest, self.include_dev) + /// Returns the number of progress steps the update will report. Pre-compiling + /// adds two per script: one to compile it, one to copy the bytecode across. + pub fn total_files(&self, pre_compile: bool) -> usize { + let files = count_files(&self.manifest_dir, &self.manifest, self.include_dev); + if pre_compile { + files + 2 * count_lua_files(&self.manifest_dir, &self.manifest, self.include_dev) + } else { + files + } } /// Execute copies the files and updates the state. + /// + /// With a compiler the package is staged and compiled locally first, so a + /// broken script fails the update before the SD card or the store is touched. pub fn execute( self, dry_run: bool, + compiler: Option<&mut dyn LuaCompiler>, mut on_file: impl FnMut(&str), ) -> Result { if self.up_to_date { @@ -182,11 +193,12 @@ impl UpdateCommand { if !dry_run { store.remove(&self.old_package.id); - let (total_copied, copied_files) = copy_content_items( + let (total_copied, copied_files) = stage_and_copy( &self.manifest, &self.manifest_dir, &sd_root, self.include_dev, + compiler, &mut on_file, )?; @@ -279,7 +291,7 @@ mod tests { store, )?; - let result = cmd.execute(dry_run, |_| {})?; + let result = cmd.execute(dry_run, None, |_| {})?; store = result.store; results.push(TestUpdateResult { package: result.package, diff --git a/tests/luac_compile.rs b/tests/luac_compile.rs new file mode 100644 index 0000000..c7b08c2 --- /dev/null +++ b/tests/luac_compile.rs @@ -0,0 +1,315 @@ +use assert_cmd::Command; +use std::path::Path; + +/// End-to-end tests for Lua bytecode compilation. +/// +/// Ignored by default because they download the EdgeTX Lua compiler WASM module +/// on first run. +/// +/// Run explicitly with: cargo test --test luac_compile -- --ignored +const SCRIPT: &str = r#" +local M = {} +function M.run(event) + local total = 0 + for i = 1, 10 do total = total + i end + lcd.drawText(1, 1, "total: " .. total, 0) + return 0 +end +return { run = M.run, init = function() end } +"#; + +fn cli() -> Command { + Command::cargo_bin("edgetx-cli").expect("find binary") +} + +/// Check the bytecode header the radio validates before loading a script. +/// +/// EdgeTX patches `ldump.c` to write `size_t` as an `int`, so every size byte is +/// 4 -- a stock 64-bit `luac` would emit 8 and the radio would reject the file. +fn assert_edgetx_bytecode(data: &[u8]) { + assert!(data.len() > 18, "bytecode is too short to hold a header"); + assert_eq!(&data[..4], b"\x1bLua", "missing Lua signature"); + assert_eq!(data[4], 0x53, "expected Lua 5.3 bytecode"); + assert_eq!(data[5], 0, "unexpected bytecode format"); + assert_eq!(&data[6..12], b"\x19\x93\r\n\x1a\n", "corrupt header data"); + assert_eq!( + &data[12..17], + &[4, 4, 4, 4, 4], + "sizes must all be 4 bytes: int, size_t-as-int, Instruction, lua_Integer, lua_Number" + ); +} + +#[test] +#[ignore] +fn compiles_a_script_to_radio_compatible_bytecode() { + let dir = tempfile::tempdir().expect("create temp dir"); + let script = dir.path().join("main.lua"); + std::fs::write(&script, SCRIPT).expect("write script"); + + cli() + .args(["dev", "luac", script.to_str().unwrap()]) + .assert() + .success(); + + let luac = dir.path().join("main.luac"); + assert!(luac.exists(), "the compiler wrote no output"); + assert_edgetx_bytecode(&std::fs::read(&luac).expect("read bytecode")); +} + +#[test] +#[ignore] +fn honours_the_output_path_and_debug_info() { + let dir = tempfile::tempdir().expect("create temp dir"); + let script = dir.path().join("main.lua"); + std::fs::write(&script, SCRIPT).expect("write script"); + + let stripped = dir.path().join("stripped.luac"); + let with_debug = dir.path().join("debug.luac"); + + cli() + .args([ + "dev", + "luac", + script.to_str().unwrap(), + "-o", + stripped.to_str().unwrap(), + ]) + .assert() + .success(); + + cli() + .args([ + "dev", + "luac", + script.to_str().unwrap(), + "-o", + with_debug.to_str().unwrap(), + "--keep-debug", + ]) + .assert() + .success(); + + let stripped_len = std::fs::metadata(&stripped).unwrap().len(); + let debug_len = std::fs::metadata(&with_debug).unwrap().len(); + assert!( + stripped_len < debug_len, + "stripping should shrink the bytecode: {stripped_len} vs {debug_len}" + ); +} + +#[test] +#[ignore] +fn reports_syntax_errors_without_writing_output() { + let dir = tempfile::tempdir().expect("create temp dir"); + let script = dir.path().join("broken.lua"); + std::fs::write(&script, "local x = 1\nif x then\n y ==== 2\nend\n").expect("write script"); + + let output = cli() + .args(["dev", "luac", script.to_str().unwrap()]) + .output() + .expect("run command"); + + assert!(!output.status.success(), "a broken script must fail"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("syntax error"), "got: {stderr}"); + assert!( + stderr.contains(":3:"), + "the error should name the line: {stderr}" + ); + assert!(!dir.path().join("broken.luac").exists()); +} + +/// A package with two scripts, which also exercises reusing one compiler across +/// several compiles (the module's output buffer is reused between calls). +fn write_package(dir: &Path) { + std::fs::create_dir_all(dir.join("SCRIPTS/TOOLS/MyTool")).unwrap(); + std::fs::create_dir_all(dir.join("SCRIPTS/FUNCTIONS")).unwrap(); + std::fs::write( + dir.join("edgetx.yml"), + "package:\n id: github.com/test/mytool\n description: Build test\n\ + tools:\n - name: MyTool\n path: SCRIPTS/TOOLS/MyTool\n\ + functions:\n - name: Logger\n path: SCRIPTS/FUNCTIONS/logger.lua\n", + ) + .unwrap(); + std::fs::write(dir.join("SCRIPTS/TOOLS/MyTool/main.lua"), SCRIPT).unwrap(); + std::fs::write( + dir.join("SCRIPTS/FUNCTIONS/logger.lua"), + "return { run = function() return 0 end }", + ) + .unwrap(); +} + +#[test] +#[ignore] +fn builds_an_installable_package_with_bytecode() { + let dir = tempfile::tempdir().expect("create temp dir"); + let src = dir.path().join("src"); + let out = dir.path().join("dist"); + let sd = dir.path().join("sdcard"); + std::fs::create_dir_all(&src).unwrap(); + std::fs::create_dir_all(&sd).unwrap(); + write_package(&src); + + cli() + .args([ + "dev", + "build", + "--src-dir", + src.to_str().unwrap(), + "--out", + out.to_str().unwrap(), + ]) + .assert() + .success(); + + assert!( + out.join("edgetx.yml").exists(), + "a package build keeps its manifest" + ); + for rel in [ + "SCRIPTS/TOOLS/MyTool/main.lua", + "SCRIPTS/TOOLS/MyTool/main.luac", + "SCRIPTS/FUNCTIONS/logger.lua", + "SCRIPTS/FUNCTIONS/logger.luac", + ] { + assert!(out.join(rel).exists(), "{rel} missing from the build"); + } + assert_edgetx_bytecode(&std::fs::read(out.join("SCRIPTS/TOOLS/MyTool/main.luac")).unwrap()); + + // The build output installs as a package, bytecode included. + cli() + .args([ + "pkg", + "install", + "--dir", + sd.to_str().unwrap(), + out.to_str().unwrap(), + ]) + .assert() + .success(); + + assert!(sd.join("SCRIPTS/TOOLS/MyTool/main.luac").exists()); + assert!(sd.join("SCRIPTS/FUNCTIONS/logger.luac").exists()); +} + +#[test] +#[ignore] +fn compresses_to_an_sd_card_archive() { + let dir = tempfile::tempdir().expect("create temp dir"); + let src = dir.path().join("src"); + let out = dir.path().join("artifacts"); + std::fs::create_dir_all(&src).unwrap(); + write_package(&src); + + cli() + .args([ + "dev", + "build", + "--src-dir", + src.to_str().unwrap(), + "--out", + out.to_str().unwrap(), + "--compress", + "--name", + "mytool-v1.2.3", + ]) + .assert() + .success(); + + let archive = out.join("mytool-v1.2.3.zip"); + assert!(archive.exists(), "no archive was produced"); + assert!( + !out.join("mytool-v1.2.3").exists(), + "the staging directory should be gone once zipped" + ); + + let file = std::fs::File::open(&archive).expect("open archive"); + let mut zip = zip::ZipArchive::new(file).expect("read archive"); + let names: Vec = (0..zip.len()) + .map(|i| zip.by_index(i).unwrap().name().to_string()) + .collect(); + + for rel in [ + "SCRIPTS/TOOLS/MyTool/main.lua", + "SCRIPTS/TOOLS/MyTool/main.luac", + "SCRIPTS/FUNCTIONS/logger.luac", + ] { + assert!( + names.iter().any(|n| n == rel), + "{rel} missing from {names:?}" + ); + } + assert!( + !names.iter().any(|n| n.ends_with("edgetx.yml")), + "an SD card archive carries no manifest: {names:?}" + ); + + // Refuse to clobber an existing archive. + cli() + .args([ + "dev", + "build", + "--src-dir", + src.to_str().unwrap(), + "--out", + out.to_str().unwrap(), + "--compress", + "--name", + "mytool-v1.2.3", + ]) + .assert() + .failure(); +} + +#[test] +#[ignore] +fn installs_with_pre_compiled_bytecode() { + let dir = tempfile::tempdir().expect("create temp dir"); + let src = dir.path().join("src"); + let sd = dir.path().join("sdcard"); + std::fs::create_dir_all(&src).unwrap(); + std::fs::create_dir_all(&sd).unwrap(); + write_package(&src); + + cli() + .args([ + "pkg", + "install", + "--dir", + sd.to_str().unwrap(), + "--pre-compile", + src.to_str().unwrap(), + ]) + .assert() + .success(); + + let luac = sd.join("SCRIPTS/TOOLS/MyTool/main.luac"); + assert!(luac.exists(), "no bytecode was installed"); + assert_edgetx_bytecode(&std::fs::read(&luac).unwrap()); + + // The radio reloads the source whenever it looks newer than the bytecode. + let lua_time = std::fs::metadata(sd.join("SCRIPTS/TOOLS/MyTool/main.lua")) + .unwrap() + .modified() + .unwrap(); + let luac_time = std::fs::metadata(&luac).unwrap().modified().unwrap(); + assert!( + luac_time >= lua_time, + "the bytecode looks stale on the card" + ); + + // Removing the package takes the generated bytecode with it. + cli() + .args([ + "pkg", + "remove", + "--dir", + sd.to_str().unwrap(), + "github.com/test/mytool", + ]) + .assert() + .success(); + + assert!(!luac.exists(), "the bytecode outlived its package"); + assert!(!sd.join("SCRIPTS/TOOLS/MyTool/main.lua").exists()); +}