From 3df3bd1d82531fad1ecc37170b83ec2c8ce11e6b Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Thu, 19 Nov 2026 10:13:50 -0800 Subject: [PATCH 1/3] Add an experimental Rust Spark Connect frontend Serve the Spark Connect protocol through an injectable execution engine, with Arrow IPC forwarding, negotiated byte chunking, session lifetime management and explicit experimental compatibility limits. --- .github/workflows/server-frontend.yml | 100 + Cargo.lock | 15 + Cargo.toml | 5 + README.md | 11 + SECURITY.md | 19 + crates/spark-connect-core/Cargo.toml | 2 +- crates/spark-connect-proto/Cargo.toml | 3 + crates/spark-connect-proto/build.rs | 4 +- crates/spark-connect-proto/src/lib.rs | 2 + crates/spark-connect-server/Cargo.toml | 41 + crates/spark-connect-server/README.md | 193 ++ crates/spark-connect-server/src/engine/mod.rs | 149 ++ crates/spark-connect-server/src/ids.rs | 154 ++ crates/spark-connect-server/src/lib.rs | 126 + crates/spark-connect-server/src/schema.rs | 231 ++ crates/spark-connect-server/src/service.rs | 1205 +++++++++ .../tests/engine_injection.rs | 2203 +++++++++++++++++ crates/spark-connect-server/tests/protocol.rs | 942 +++++++ .../tests/schema_consistency.rs | 481 ++++ .../tests/server_shutdown.rs | 192 ++ .../tests/support/arrow.rs | 112 + .../spark-connect-server/tests/support/mod.rs | 196 ++ .../tests/support/requests.rs | 126 + .../tests/support/server.rs | 87 + dev/design/ci.md | 18 +- 25 files changed, 6613 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/server-frontend.yml create mode 100644 crates/spark-connect-server/Cargo.toml create mode 100644 crates/spark-connect-server/README.md create mode 100644 crates/spark-connect-server/src/engine/mod.rs create mode 100644 crates/spark-connect-server/src/ids.rs create mode 100644 crates/spark-connect-server/src/lib.rs create mode 100644 crates/spark-connect-server/src/schema.rs create mode 100644 crates/spark-connect-server/src/service.rs create mode 100644 crates/spark-connect-server/tests/engine_injection.rs create mode 100644 crates/spark-connect-server/tests/protocol.rs create mode 100644 crates/spark-connect-server/tests/schema_consistency.rs create mode 100644 crates/spark-connect-server/tests/server_shutdown.rs create mode 100644 crates/spark-connect-server/tests/support/arrow.rs create mode 100644 crates/spark-connect-server/tests/support/mod.rs create mode 100644 crates/spark-connect-server/tests/support/requests.rs create mode 100644 crates/spark-connect-server/tests/support/server.rs diff --git a/.github/workflows/server-frontend.yml b/.github/workflows/server-frontend.yml new file mode 100644 index 000000000..85cc57e9c --- /dev/null +++ b/.github/workflows/server-frontend.yml @@ -0,0 +1,100 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Server frontend CI + +on: + pull_request: + branches: [master, "branch-*"] + paths: + - ".github/workflows/server-frontend.yml" + - "Cargo.toml" + - "Cargo.lock" + - ".cargo/**" + - "rust-toolchain*" + - "crates/spark-connect-server/**" + - "crates/spark-connect-proto/**" + push: + branches: [master, "branch-*"] + paths: + - ".github/workflows/server-frontend.yml" + - "Cargo.toml" + - "Cargo.lock" + - ".cargo/**" + - "rust-toolchain*" + - "crates/spark-connect-server/**" + - "crates/spark-connect-proto/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_BUILD_JOBS: "2" + CARGO_PROFILE_TEST_DEBUG: "0" + # Strict linting uses the pinned toolchain in lint.yml. + RUSTFLAGS: "" + +jobs: + frontend: + name: Frontend (Rust ${{ matrix.rust-version }}) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + rust-version: [stable, "1.88.0"] + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + # Use runner-provided rustup to keep actions limited to actions/*. + run: | + rustup toolchain install ${{ matrix.rust-version }} --profile minimal + rustup default ${{ matrix.rust-version }} + + - name: Cache cargo and target + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-${{ runner.arch }}-cargo-server-frontend-${{ matrix.rust-version }}-${{ hashFiles('Cargo.lock') }} + restore-keys: ${{ runner.os }}-${{ runner.arch }}-cargo-server-frontend-${{ matrix.rust-version }}- + + - name: Install protobuf compiler + # Ubuntu 24.04 provides protoc with proto3 optional support (3.15+). + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + protoc --version + + - name: Test frontend + # Protocol and injected-engine tests use local fixtures without an execution backend. + run: cargo test --locked -p spark-connect-server + + - name: Check protocol feature combinations + run: | + cargo check --locked -p apache-spark-connect-proto + cargo check --locked -p apache-spark-connect-proto --features server diff --git a/Cargo.lock b/Cargo.lock index 40cf5dabb..41a9b1b3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4048,6 +4048,21 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spark-connect-server" +version = "0.1.0" +dependencies = [ + "apache-spark-connect-proto", + "arrow", + "futures", + "h2", + "prost", + "tokio", + "tokio-stream", + "tonic", + "uuid", +] + [[package]] name = "sqlparser" version = "0.60.0" diff --git a/Cargo.toml b/Cargo.toml index fb6261a14..189b8bab6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "crates/spark-connect-build", "crates/pyspark-rs", "examples", + "crates/spark-connect-server", ] exclude = [ "examples/wasm-udfs", @@ -44,6 +45,8 @@ exclude = [ # (so `cargo test --workspace` stays green without it). Build them explicitly with # `cargo build --manifest-path examples/wasm-udfs/Cargo.toml` (etc) in CI or when # the target is installed locally. +# The experimental protocol frontend is opt-in; build or test it with +# `-p spark-connect-server`. It is included in workspace lint checks. default-members = [ "crates/spark-connect-proto", "crates/spark-connect-core", @@ -87,11 +90,13 @@ should_implement_trait = "allow" tonic = { version = "0.14", default-features = false, features = ["channel", "codegen"] } tonic-prost = "0.14" tonic-prost-build = "0.14" +h2 = "0.4" prost = "0.14" prost-types = "0.14" # async runtime tokio = { version = "1", features = ["rt-multi-thread", "sync", "macros", "time"] } +tokio-stream = "0.1" futures = "0.3" # Arrow (result decoding). diff --git a/README.md b/README.md index d75af9868..fcc755db1 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,17 @@ fn main() -> Result<(), Box> { See the [WASM UDF guide](https://apache.github.io/spark-connect-rust/udfs/) for the full setup, SQL registration, supported types, and more. +## Experimental server frontend + +The unpublished [`spark-connect-server`](crates/spark-connect-server/README.md) +library provides a Rust Spark Connect protocol frontend. It manages protocol sessions, +operations and Arrow IPC responses, delegating planning and execution to an +application-provided `Engine`. The library includes no execution backend or standalone +server executable; its protocol tests use injected engine fixtures. + +The frontend is intended for local, trusted experiments. Read its +[security model](SECURITY.md#experimental-server-frontend) before embedding it. + ## Contributing Issues are tracked in ASF JIRA under diff --git a/SECURITY.md b/SECURITY.md index 80e532538..d80df8b40 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -25,3 +25,22 @@ material, TLS validation) inherit from that model. Automated security scanning agents should consult that document for the project's in-scope / out-of-scope declarations before reporting issues. + +## Experimental server frontend + +The optional `spark-connect-server` library is an experimental protocol +frontend for local, trusted workloads. It does not provide authentication, +authorization or TLS. Session IDs, operation IDs and client-provided user +context identify protocol state; they do not establish a caller's identity +or access rights. + +Bind its listener only to loopback. The embedding application supplies the +listener and the `Engine` implementation; the library does not enforce the +listener's network scope. Any process that can reach it can submit requests +to that engine. Protocol limits and session isolation are resource and +correctness guards, not a security boundary between mutually untrusted users. + +This frontend does not include an execution backend or a standalone server. +Its injected-engine tests establish frontend protocol behavior, not safe +exposure of an execution engine to untrusted clients. Apache Spark's operator +security guidance remains relevant to any application that connects an engine. diff --git a/crates/spark-connect-core/Cargo.toml b/crates/spark-connect-core/Cargo.toml index a34fb8980..9d4e6c9da 100644 --- a/crates/spark-connect-core/Cargo.toml +++ b/crates/spark-connect-core/Cargo.toml @@ -32,7 +32,7 @@ prost-types.workspace = true crc32fast = "1" sha2 = "0.10" # Transport errors that tonic keeps as a status's source, for `RetryPolicy::can_retry`. -h2 = "0.4" +h2.workspace = true hyper = "1" # The Basic credential and the percent-decoded userinfo of a proxy URI, already built for tonic. base64 = "0.22" diff --git a/crates/spark-connect-proto/Cargo.toml b/crates/spark-connect-proto/Cargo.toml index efc4c8333..330fdfa04 100644 --- a/crates/spark-connect-proto/Cargo.toml +++ b/crates/spark-connect-proto/Cargo.toml @@ -15,6 +15,9 @@ workspace = true [lib] name = "spark_connect_proto" +[features] +server = [] + [dependencies] tonic.workspace = true tonic-prost.workspace = true diff --git a/crates/spark-connect-proto/build.rs b/crates/spark-connect-proto/build.rs index 048ae15f0..dd85bafee 100644 --- a/crates/spark-connect-proto/build.rs +++ b/crates/spark-connect-proto/build.rs @@ -35,8 +35,10 @@ fn main() -> Result<(), Box> { let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").ok_or("OUT_DIR is not set")?); let full_set = out_dir.join("spark_connect_descriptor_set_full.bin"); tonic_prost_build::configure() - .build_server(false) + .build_server(cfg!(feature = "server")) .build_client(true) + // async_trait repeats Future's must_use on generated server methods. + .trait_attribute("SparkConnectService", "#[allow(clippy::double_must_use)]") .bytes(".") .file_descriptor_set_path(&full_set) .compile_protos(&protos, &[proto_root])?; diff --git a/crates/spark-connect-proto/src/lib.rs b/crates/spark-connect-proto/src/lib.rs index 35689ba44..58eb65a94 100644 --- a/crates/spark-connect-proto/src/lib.rs +++ b/crates/spark-connect-proto/src/lib.rs @@ -3,6 +3,8 @@ //! All messages live in the `spark.connect` protobuf package and are re-exported //! here at the crate root. The generated gRPC client is //! [`spark_connect_service_client::SparkConnectServiceClient`]. +//! Enable the optional `server` feature for `spark_connect_service_server` +//! bindings. Embeddings configure tonic's transport server features separately. //! //! Baseline protocol version: see `proto/PROTO_VERSION.txt` / `proto/SPARK_SHA.txt`. diff --git a/crates/spark-connect-server/Cargo.toml b/crates/spark-connect-server/Cargo.toml new file mode 100644 index 000000000..96adcd4a1 --- /dev/null +++ b/crates/spark-connect-server/Cargo.toml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "spark-connect-server" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false +description = "Experimental Spark Connect server frontend with an injectable execution engine" + +[lints] +workspace = true + +[dependencies] +spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto", features = ["server"] } +arrow.workspace = true +futures.workspace = true +prost.workspace = true +tokio = { workspace = true, features = ["net"] } +tokio-stream.workspace = true +tonic = { workspace = true, features = ["server", "router"] } +uuid.workspace = true + +[dev-dependencies] +h2.workspace = true diff --git a/crates/spark-connect-server/README.md b/crates/spark-connect-server/README.md new file mode 100644 index 000000000..e59a8dd07 --- /dev/null +++ b/crates/spark-connect-server/README.md @@ -0,0 +1,193 @@ + + +# Experimental Spark Connect server frontend + +This unpublished library serves a subset of the Spark Connect protocol using the +workspace's published `apache-spark-connect-proto` messages and its optional +`server` feature. This additive feature exposes generated server bindings in the +protocol crate's public API; Cargo unifies it with other uses of that crate in +the workspace. The caller supplies an +`Engine` implementation and a bound Tokio TCP listener to `serve_with_engine`. +Use `serve_with_engine_and_options` with `ServerOptions` to configure Arrow +chunking. +No execution engine or standalone server executable is included. + +Rust owns protocol sessions, operation IDs, response framing and result-stream +lifetimes. Each protocol session receives an `EngineSession` that owns its +planning, catalog and configuration state. Its `prepare` method returns a +`PreparedQuery` or command response payloads; the frontend adds protocol IDs and +`ResultComplete` for requests that opt into reattachment. The engine decides +which plans, commands and metadata requests it can execute and reports +unsupported inputs explicitly. + +## Protocol subset + +The frontend routes `ExecutePlan`, `AnalyzePlan` and `Config` to the engine. It +implements `ReleaseExecute` and `ReleaseSession`, including operation cancellation +through `ReleaseAll` and session recreation when `allow_reconnect` permits it. +Completed responses are not retained. Reattachment, artifact upload/status, +`Interrupt`, enriched error details, session cloning and status queries return +`UNIMPLEMENTED`. + +Session creation and `ExecutePlan` operation IDs follow JDK 21 `UUID.fromString` +parsing, including uppercase and shortened hyphen-separated groups. Session IDs +retain their supplied spelling: equivalent UUID spellings identify different +sessions, as in Spark. `ExecutePlan` normalizes operation IDs to canonical +36-character lowercase UUIDs and rejects duplicate active IDs within a session. +Completed IDs are forgotten and can be reused, so retrying a completed command +with the same ID can execute it again. Spark remembers recently completed and +abandoned operation IDs and rejects their reuse. +`ReleaseExecute` looks up its operation ID as supplied; clients should use the +normalized ID returned by execution. It ignores the client-observed server +session ID, as Spark does. `ExecutePlan`, `AnalyzePlan` and `Config` check that +observed ID. Client user IDs distinguish session keys but are not authenticated +identities. Release RPCs look up session keys without parsing UUIDs. +`ReleaseSession` for an unknown session is a no-op. `ReleaseExecute` returns +`NOT_FOUND` for a missing session and `FAILED_PRECONDITION` while its +released-session tombstone is retained. + +The frontend validates execution operation IDs and plan envelopes before resolving +a session, so invalid execution requests allocate no backend. Spark resolves the +session first; this ordering can change which error is reported when multiple +fields are invalid. + +`ResultComplete` is sent only when a request sets `ReattachOptions.reattachable`. +That option supports completion detection on the single live result stream; +results cannot be resumed or replayed because `ReattachExecute` is unimplemented. +Nonreattachable requests complete when their result stream ends. + +`ReleaseAll` cancels active operations, including nonreattachable operations. +This is a deliberate experimental deviation: Spark rejects release of a +nonreattachable operation with `INVALID_CURSOR.NOT_REATTACHABLE` and uses +`Interrupt` for cancellation. This frontend uses `ReleaseAll` for cancellation +while `Interrupt` remains unimplemented. `ReleaseUntil` and requests without a +release kind are no-ops, including for live operations; Spark validates +reattachability and the release kind for a live operation. Execution metrics and +Spark structured error classes are also absent; failures use gRPC status codes +and messages. + +Protobuf nesting follows the selected Prost features; dependency feature +unification can disable its recursion limit, and the frontend adds no independent +nesting guard. Deep plans can fail before engine planning; when its recursion +limit is enabled, the Rust transport reports a protobuf recursion error as +`INTERNAL`. + +## Arrow results + +A prepared query declares an Arrow schema before execution. The frontend converts +supported types to the Spark schema representation and checks each batch against +that declaration. Batches may use Spark-equivalent Arrow representations and +nullability different from the declaration. Within one result, emitted IPC +batches must share physical types, field names and nullability, so clients can +assemble them into one Arrow table. Metadata may differ between batches. +Declared field metadata and unsupported types fail explicitly. + +Engines may yield Arrow record batches or complete Arrow IPC streams containing +one batch and its actual nonnegative row count. The frontend encodes each record +batch once and forwards existing IPC bytes unchanged, including compressed IPC. +It reads the IPC schema for validation and trusts the producer's batch contents, +record-batch count, EOS marker and declared row count; it does not validate those +parts of the stream. Record-batch encoding materializes string/binary view +arrays, including nested children, to `LargeUtf8`/`LargeBinary`. Existing IPC +retains its physical types. The frontend reports `INTERNAL` before sending a +batch whose output schema differs from earlier batches; engines mixing record +batches with IPC must use consistent output representations. + +When the client sets `ResultChunkingOptions.allow_arrow_batch_chunking`, the +frontend divides serialized IPC bytes into chunks and sets `chunk_index` and +`num_chunks_in_batch`. Boundaries need not align with rows or Arrow buffers; the +client concatenates a batch's chunks before decoding or decompressing it. A +preferred chunk payload size between 1 KiB and the configured maximum is accepted; +otherwise the maximum is used. `ServerOptions.max_arrow_chunk_size` defaults to +120,795,955 bytes (90% of 128 MiB), matching the JVM server's default chunk payload +limit. It limits IPC payload bytes, excluding the protobuf envelope. Setting it +to zero disables chunking even when the client opts in. An opted-in client must +accept the configured maximum plus its protobuf envelope, or request a smaller +chunk size. A 128 MiB receive limit accommodates the default maximum; raw tonic's +4 MiB default does not accommodate chunks above 4 MiB. + +Without client opt-in, or when chunking is disabled, each IPC batch is sent whole. +Engines remain responsible for bounding batch production, including fitting the +client's gRPC receive limit when batches are sent whole. The frontend applies no +128 MiB batch-production cap. Spark's JVM producer separately uses +`spark.sql.execution.arrow.maxRecordsPerBatch` and a byte estimate of 70% of +`spark.connect.grpc.arrow.maxBatchSize` (128 MiB by default); its estimate can +overshoot by a row. Generic engines must choose their own production bounds. +Schema and command response envelopes have a separate 3 MiB limit and fail with +`RESOURCE_EXHAUSTED` when oversized. Each batch carries a cumulative row offset, +and empty results still include an empty batch. + +## Lifetime and trust requirements + +Engine methods and streams must cancel their owned work when dropped. The +frontend cancels them when a client disconnects, an operation is released, a +session is released, server shutdown begins or the serving future is dropped. +Results pass through a bounded +channel, so unread responses eventually stop polling the engine. Blocking Arrow +encoding retains operation ownership until its worker finishes. IPC chunks share +their batch's backing allocation, which remains live while any chunk references +it. Larger batches and chunk limits can therefore increase retained memory even +though the response channel is bounded. + +`ReleaseAll` waits up to two seconds for teardown and reports `DEADLINE_EXCEEDED` +if it is still draining. Shutdown cancels sessions and waits up to two seconds +for transport drain; returning from `serve_with_engine` does not prove that an +engine's independently scheduled or blocking work has finished. The caller must +honor the engine's cleanup contract before releasing its resources. + +`ReleaseSession(allow_reconnect=true)` permits a new backend session with the +same protocol ID while the previous incarnation still drains. Engines must keep +the two incarnations' resources independent. Commands may commit effects before +frontend response validation fails, for example on an oversized payload; that +failure provides no rollback or safe-retry guarantee. + +With panic unwinding enabled, panics in `prepare`, `analyze`, `config` and query +execution are translated to `INTERNAL`. Synchronous session creation and query +schema access are outside that boundary and must not panic. Resource cleanup +must not panic either. + +This is an experiment for trusted local clients and trusted engine outputs. +Keep the supplied listener on loopback: this frontend supplies no authentication, +authorization or TLS. Requests can be up to 128 MiB, matching Spark's default +inbound message limit. Larger messages receive tonic's `OUT_OF_RANGE` status. +Active sessions and operations +have no admission limit or idle expiry; only released-session tombstones are +bounded, to the most recent 4096 distinct IDs. Spark instead uses a size-bounded +cache configured by `spark.connect.session.manager.closedSessionsTombstonesSize`, +which defaults to 1000; this frontend's fixed bound and release-order eviction +policy differ. An evicted ID can create a new session. These limits do not +provide isolation or complete memory and CPU quotas. +Execution permissions and resource access are the engine's responsibility. + +## Build and test + +Use the workspace Rust toolchain and a `protoc` compiler with proto3 optional +support. From the repository root: + +```bash +cargo test --locked -p spark-connect-server +cargo clippy --locked -p spark-connect-server --all-targets --no-deps -- -D warnings +``` + +The tests run a real gRPC frontend against deterministic injected engines. They +cover wire framing, schema and IPC handling, backend error propagation, +backpressure, operation ownership, cancellation and session lifetime. Fixture +relations select predefined outputs; they do not implement Spark operators or +establish query semantics or Spark server compatibility. diff --git a/crates/spark-connect-server/src/engine/mod.rs b/crates/spark-connect-server/src/engine/mod.rs new file mode 100644 index 000000000..21cfbf266 --- /dev/null +++ b/crates/spark-connect-server/src/engine/mod.rs @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Native engine boundary for Spark Connect planning and execution. +//! +//! The frontend owns protocol IDs, operation lifetime, framing and backpressure. +//! Engines own their catalog, configuration and native plans. This Rust API is +//! an initial in-process contract for an injected execution engine. + +use std::pin::Pin; +use std::sync::Arc; + +use arrow::datatypes::SchemaRef; +use arrow::record_batch::RecordBatch; +use futures::Stream; +use spark_connect_proto as proto; +use tonic::Status; + +/// Convert an engine Arrow schema into the Spark Connect schema representation. +pub use crate::schema::spark_schema; + +/// Protocol context for a new backend session. +#[derive(Clone, Debug)] +pub struct SessionContext { + pub session_id: String, + /// Client-supplied metadata, not an authenticated identity. + pub user_context: Option, +} + +/// Factory for sessions bound to one native engine. +pub trait Engine: Send + Sync { + /// Allocate local session state without blocking, performing I/O or panicking. + /// Defer asynchronous initialization to the session methods. The frontend + /// serializes creation to ensure one backend session per active incarnation. + /// A reconnecting session release permits a replacement with the same + /// `session_id` while references to the previous incarnation are still + /// draining. The engine must keep those incarnations' resources independent. + fn create_session(&self, context: SessionContext) -> Result, Status>; +} + +/// Backend-owned catalog, configuration and planning state. +/// +/// Dropping a method future must cancel its work. Dropping the final session +/// reference releases its resources after in-flight operations have finished. +/// Resource cleanup must not panic. +/// With unwinding enabled, panics in these methods become `INTERNAL` errors; +/// the frontend keeps the session available for later requests. Implementations +/// must leave their state usable when unwinding. +// async_trait repeats Future's must_use annotation on generated trait methods. +#[allow(clippy::double_must_use)] +#[tonic::async_trait] +pub trait EngineSession: Send + Sync { + /// Prepare a query, or perform an eager protocol command. The frontend + /// reserves the operation ID before calling this method. + /// If cancelled, a command must leave its state change fully committed or + /// rolled back, with no partially applied changes. + async fn prepare(&self, operation: proto::plan::OpType) -> Result; + + async fn analyze( + &self, + input: proto::analyze_plan_request::Analyze, + ) -> Result; + + async fn config( + &self, + input: proto::config_request::operation::OpType, + ) -> Result; +} + +/// Prepared native query or command-specific response payloads. +pub enum PreparedOperation { + Query(Box), + /// The frontend adds envelope IDs and, for requests that opt into + /// reattachment, the final `ResultComplete` message. + /// Eager command effects may already be committed when later frontend + /// response validation fails, for example when a payload exceeds the size + /// limit. Such a failure does not roll back effects or make retries safe. + Command(Vec), +} + +/// Query execution starts after the frontend validates and sends its schema. +// async_trait repeats Future's must_use annotation on generated trait methods. +#[allow(clippy::double_must_use)] +#[tonic::async_trait] +pub trait PreparedQuery: Send { + /// The declared Spark schema. This synchronous accessor must not panic. + /// Batches must have the same field names and Spark-visible types; + /// equivalent Arrow representations are accepted. + /// Batch nullability and metadata may differ from this declaration, but + /// emitted IPC batches in one result must share physical types, field names + /// and nullability. Metadata may differ between batches. Record-batch encoding + /// materializes view arrays to large-offset types; existing IPC retains its + /// physical types. Mixing incompatible output representations fails before + /// the changed output schema is sent. Declared fields must not contain + /// metadata, which Spark schema conversion rejects. + fn schema(&self) -> SchemaRef; + + /// Dropping this future or its returned stream must cancel backend work; + /// implementations must not leave detached execution tasks running. + async fn execute(self: Box) -> Result; +} + +/// Engine output, optionally already serialized for Spark Connect. +/// +/// Engines must bound batch production. Optional byte chunking bounds network +/// payloads, not the complete batch's allocation. When chunking is disabled or +/// the client does not opt in, each batch must fit the client's gRPC receive limit. +pub enum ResultBatch { + /// Encoded once as a complete IPC stream before optional byte chunking. + Arrow(RecordBatch), + /// A complete IPC stream containing its schema, exactly one record batch, + /// and EOS. The producer must supply that batch's actual nonnegative row count. + /// The frontend validates the schema and trusts the producer's batch contents + /// and declared row count. IPC bytes, including compression, are preserved. + /// Clients that opt in receive byte chunks for reassembly before decoding; + /// otherwise the whole stream is sent, so producers must respect the client's + /// gRPC receive limit when selecting batch sizes. + Ipc { data: Vec, row_count: i64 }, +} + +impl From for ResultBatch { + fn from(batch: RecordBatch) -> Self { + Self::Arrow(batch) + } +} + +/// Pull-based results: the frontend polls only as client backpressure allows. +pub type BatchStream = Pin> + Send>>; + +/// Engine configuration output without frontend-owned protocol IDs. +#[derive(Default)] +pub struct ConfigResult { + pub pairs: Vec, + pub warnings: Vec, +} diff --git a/crates/spark-connect-server/src/ids.rs b/crates/spark-connect-server/src/ids.rs new file mode 100644 index 000000000..3c9c3b1b4 --- /dev/null +++ b/crates/spark-connect-server/src/ids.rs @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use uuid::Uuid; + +/// Matches `UUID.fromString` on OpenJDK 21, including its +/// abbreviated groups, optional plus signs and truncation of oversized fields. +pub(crate) fn parse_uuid(value: &str) -> Option { + if value.encode_utf16().take(37).count() > 36 { + return None; + } + let mut groups = value.split('-'); + let mut fields = [0_i64; 5]; + for field in &mut fields { + let group = groups.next()?; + let digits = group.strip_prefix('+').unwrap_or(group); + if digits.is_empty() { + return None; + } + *field = digits.chars().try_fold(0_i64, |value, digit| { + value + .checked_mul(16)? + .checked_add(i64::from(hex_digit(digit)?)) + })?; + } + if groups.next().is_some() { + return None; + } + Some(Uuid::from_u128( + ((fields[0] as u128 & 0xffff_ffff) << 96) + | ((fields[1] as u128 & 0xffff) << 80) + | ((fields[2] as u128 & 0xffff) << 64) + | ((fields[3] as u128 & 0xffff) << 48) + | (fields[4] as u128 & 0xffff_ffff_ffff), + )) +} + +fn hex_digit(digit: char) -> Option { + if let Some(value) = digit.to_digit(16) { + return Some(value); + } + match digit { + '\u{ff21}'..='\u{ff26}' => Some(digit as u32 - 0xff21 + 10), + '\u{ff41}'..='\u{ff46}' => Some(digit as u32 - 0xff41 + 10), + _ => { + // BMP decimal-zero code points accepted by Character.digit(char, 16) + // in OpenJDK 21 (Unicode 15.0). Java parses UTF-16 chars here, so + // supplementary-plane digits are deliberately excluded. + const ZEROES: &[u32] = &[ + 0x0660, 0x06f0, 0x07c0, 0x0966, 0x09e6, 0x0a66, 0x0ae6, 0x0b66, 0x0be6, 0x0c66, + 0x0ce6, 0x0d66, 0x0de6, 0x0e50, 0x0ed0, 0x0f20, 0x1040, 0x1090, 0x17e0, 0x1810, + 0x1946, 0x19d0, 0x1a80, 0x1a90, 0x1b50, 0x1bb0, 0x1c40, 0x1c50, 0xa620, 0xa8d0, + 0xa900, 0xa9d0, 0xa9f0, 0xaa50, 0xabf0, 0xff10, + ]; + ZEROES + .iter() + .find_map(|&zero| (digit as u32).checked_sub(zero).filter(|&n| n < 10)) + } + } +} + +#[cfg(test)] +mod tests { + use super::parse_uuid; + + #[test] + fn accepts_jdk_uuid_spellings() { + for value in [ + "1-2-3-4-5", + "+1-+2-+3-+4-+5", + "00000000000000001-2-3-4-5", + "0000000000000000000000000001-2-3-4-5", + "100000001-10002-3-4-5", + "1-2-3-4-5", + "١-٢-٣-٤-٥", + ] { + assert_eq!( + parse_uuid(value).unwrap().to_string(), + "00000001-0002-0003-0004-000000000005", + "{value}" + ); + } + assert_eq!( + parse_uuid("123E4567-E89B-12D3-A456-426614174000") + .unwrap() + .to_string(), + "123e4567-e89b-12d3-a456-426614174000" + ); + assert_eq!( + parse_uuid("A-B-C-D-E").unwrap().to_string(), + "0000000a-000b-000c-000d-00000000000e" + ); + } + + #[test] + fn masks_each_field_after_parsing_a_signed_long() { + for (value, expected) in [ + ( + "7fffffffffffffff-2-3-4-5", + "ffffffff-0002-0003-0004-000000000005", + ), + ( + "1-10002-10003-10004-5", + "00000001-0002-0003-0004-000000000005", + ), + ( + "1-2-3-4-7fffffffffffffff", + "00000001-0002-0003-0004-ffffffffffff", + ), + ] { + assert_eq!(parse_uuid(value).unwrap().to_string(), expected); + } + } + + #[test] + fn rejects_forms_the_jdk_does_not_parse() { + for value in [ + "", + "1-2-3-4", + "1-2-3-4-5-6", + "-1-2-3-4-5", + "1--2-3-4-5", + "1-2-3-4-", + "1-2-3-4-+", + "1-2-3-4-++5", + "1-2-3-4- 5", + "1-2-3-4-0x5", + "8000000000000000-2-3-4-5", + "1-2-3-4-8000000000000000", + "00000000000000000000000000001-2-3-4-5", + "123e4567e89b12d3a456426614174000", + "{123e4567-e89b-12d3-a456-426614174000}", + "urn:uuid:123e4567-e89b-12d3-a456-426614174000", + "1-2-3-4-⑤", + "1-2-3-4-𝟝", + ] { + assert!(parse_uuid(value).is_none(), "{value}"); + } + } +} diff --git a/crates/spark-connect-server/src/lib.rs b/crates/spark-connect-server/src/lib.rs new file mode 100644 index 000000000..6d2f95d84 --- /dev/null +++ b/crates/spark-connect-server/src/lib.rs @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod engine; +mod ids; +mod schema; +mod service; + +/// Generated server bindings for custom embedding. This experimental API follows +/// the protocol schema and may change with the protocol. +pub mod grpc { + pub use spark_connect_proto::spark_connect_service_server; +} + +/// Transport options for the experimental frontend. +/// Start from [`ServerOptions::default()`] and assign the desired fields. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct ServerOptions { + /// Maximum Arrow chunk payload in bytes, excluding the protobuf envelope. + /// Defaults to 90% of 128 MiB, as in Spark's JVM server. A client can request + /// a smaller size. Zero disables chunking, even when the client opts in. + /// Clients must accept this payload plus the protobuf envelope: a 128 MiB + /// receive limit accommodates the default; tonic's 4 MiB default does not. + /// Alternatively, clients can request a smaller chunk size. + /// Whole batches sent without chunking are not bounded by this option. + pub max_arrow_chunk_size: usize, +} + +impl Default for ServerOptions { + fn default() -> Self { + Self { + max_arrow_chunk_size: 128 * 1024 * 1024 * 9 / 10, + } + } +} + +struct CancelOnDrop(std::sync::Arc); + +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.cancel_all(); + } +} + +fn tcp_incoming(listener: tokio::net::TcpListener) -> tonic::transport::server::TcpIncoming { + // Builder socket options do not apply to serve_with_incoming_shutdown. + tonic::transport::server::TcpIncoming::from(listener).with_nodelay(Some(true)) +} + +/// Serve with an injected execution engine and a fresh session registry. +/// +/// The frontend binds each protocol session to the engine session created for it. +/// The caller owns the engine and signals shutdown before releasing its resources. +/// Dropping this future also cancels registered engine work, without waiting for +/// the bounded transport drain used when shutdown is signalled. +pub async fn serve_with_engine( + listener: tokio::net::TcpListener, + engine: std::sync::Arc, + shutdown: impl std::future::Future + Send + 'static, +) -> Result<(), tonic::transport::Error> { + serve_with_engine_and_options(listener, engine, ServerOptions::default(), shutdown).await +} + +/// Serve with an injected engine and explicit transport options. +/// +/// Session lifetime and shutdown follow [`serve_with_engine`]. Producers should +/// bound engine batches independently of this transport chunk size. Clients must +/// allow protobuf overhead in their gRPC receive limits. +pub async fn serve_with_engine_and_options( + listener: tokio::net::TcpListener, + engine: std::sync::Arc, + options: ServerOptions, + shutdown: impl std::future::Future + Send + 'static, +) -> Result<(), tonic::transport::Error> { + let service = std::sync::Arc::new(service::ConnectService::new(engine, options)); + let _cancel_on_drop = CancelOnDrop(service.clone()); + let router = tonic::transport::Server::builder().add_service( + grpc::spark_connect_service_server::SparkConnectServiceServer::from_arc(service.clone()) + .max_decoding_message_size(128 * 1024 * 1024), + ); + let (draining, drain_started) = tokio::sync::oneshot::channel(); + let server = router.serve_with_incoming_shutdown(tcp_incoming(listener), async move { + shutdown.await; + service.cancel_all(); + let _ = draining.send(()); + }); + tokio::pin!(server); + tokio::select! { + result = &mut server => result, + _ = drain_started => { + // A peer that stops reading must not prevent local process shutdown. + tokio::time::timeout(std::time::Duration::from_secs(2), &mut server) + .await.unwrap_or(Ok(())) + } + } +} + +#[cfg(test)] +mod tests { + use futures::StreamExt; + + #[tokio::test] + async fn accepted_connections_disable_nagle() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let mut incoming = super::tcp_incoming(listener); + let _client = tokio::net::TcpStream::connect(address).await.unwrap(); + let accepted = incoming.next().await.unwrap().unwrap(); + assert!(accepted.nodelay().unwrap()); + } +} diff --git a/crates/spark-connect-server/src/schema.rs b/crates/spark-connect-server/src/schema.rs new file mode 100644 index 000000000..daddedbbe --- /dev/null +++ b/crates/spark-connect-server/src/schema.rs @@ -0,0 +1,231 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The supported Spark schema boundary. Unsupported Arrow types fail explicitly. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, TimeUnit}; +use prost::Message; +use spark_connect_proto as proto; +use tonic::Status; + +pub(crate) const MAX_METADATA_RESPONSE_BYTES: usize = 3 * 1024 * 1024; + +pub(crate) fn result_schema(schema: &Schema) -> Result { + let schema = spark_schema(schema)?; + validate_schema_size(&schema)?; + Ok(schema) +} + +pub(crate) fn validate_schema_size(schema: &proto::DataType) -> Result<(), Status> { + if schema.encoded_len() > MAX_METADATA_RESPONSE_BYTES { + return Err(Status::resource_exhausted( + "Spark schema exceeds the 3 MiB schema/command response limit", + )); + } + Ok(()) +} + +/// Transform fields after recursively transforming their supported child fields. +pub(crate) fn map_fields( + fields: &Fields, + transform: &impl Fn(&FieldRef, DataType) -> FieldRef, +) -> Fields { + fn field(input: &FieldRef, transform: &impl Fn(&FieldRef, DataType) -> FieldRef) -> FieldRef { + let data_type = match input.data_type() { + DataType::List(child) => DataType::List(field(child, transform)), + DataType::LargeList(child) => DataType::LargeList(field(child, transform)), + DataType::Struct(children) => DataType::Struct(map_fields(children, transform)), + DataType::Map(entries, sorted) => DataType::Map(field(entries, transform), *sorted), + other => other.clone(), + }; + transform(input, data_type) + } + + fields.iter().map(|input| field(input, transform)).collect() +} + +/// Compare Spark-visible fields and types, allowing equivalent Arrow physical +/// representations and differences in nullability or field/schema metadata. +pub(crate) fn comparable_schema(schema: &Schema) -> Result { + let fields = map_fields(schema.fields(), &|field, data_type| { + Arc::new(Field::new(field.name(), data_type, true)) + }); + spark_schema(&Schema::new(fields)) +} + +pub fn spark_schema(schema: &Schema) -> Result { + spark_type(&DataType::Struct(schema.fields().clone())) +} + +fn validate_field_metadata(field: &Field) -> Result<(), Status> { + if !field.metadata().is_empty() { + return Err(Status::unimplemented( + "Arrow field metadata in Spark schemas", + )); + } + Ok(()) +} + +fn spark_field(field: &Field) -> Result { + validate_field_metadata(field)?; + Ok(proto::data_type::StructField { + name: field.name().clone(), + data_type: Some(spark_type(field.data_type())?), + nullable: field.is_nullable(), + metadata: None, + }) +} + +fn spark_type(data_type: &DataType) -> Result { + use proto::data_type::{self as pb, Kind}; + let kind = match data_type { + DataType::Null => Kind::Null(pb::Null::default()), + DataType::Boolean => Kind::Boolean(pb::Boolean::default()), + DataType::Int8 => Kind::Byte(pb::Byte::default()), + DataType::Int16 => Kind::Short(pb::Short::default()), + DataType::Int32 => Kind::Integer(pb::Integer::default()), + DataType::Int64 => Kind::Long(pb::Long::default()), + DataType::Float32 => Kind::Float(pb::Float::default()), + DataType::Float64 => Kind::Double(pb::Double::default()), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Kind::String(pb::String { + collation: "UTF8_BINARY".into(), + ..Default::default() + }), + DataType::Binary | DataType::LargeBinary | DataType::BinaryView => { + Kind::Binary(pb::Binary::default()) + } + DataType::Date32 => Kind::Date(pb::Date::default()), + DataType::Timestamp(TimeUnit::Microsecond, None) => { + Kind::TimestampNtz(pb::TimestampNtz::default()) + } + DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => { + Kind::Timestamp(pb::Timestamp::default()) + } + DataType::Decimal128(precision, scale) => Kind::Decimal(pb::Decimal { + precision: Some(i32::from(*precision)), + scale: Some(i32::from(*scale)), + ..Default::default() + }), + DataType::List(field) | DataType::LargeList(field) => { + validate_field_metadata(field)?; + Kind::Array(Box::new(pb::Array { + element_type: Some(Box::new(spark_type(field.data_type())?)), + contains_null: field.is_nullable(), + ..Default::default() + })) + } + DataType::Struct(fields) => Kind::Struct(pb::Struct { + fields: fields + .iter() + .map(|field| spark_field(field)) + .collect::>()?, + ..Default::default() + }), + DataType::Map(entries, _) => { + validate_field_metadata(entries)?; + let DataType::Struct(fields) = entries.data_type() else { + return Err(Status::invalid_argument( + "Arrow map entries must be a struct", + )); + }; + if fields.len() != 2 { + return Err(Status::invalid_argument( + "Arrow map entries need key and value fields", + )); + } + validate_field_metadata(&fields[0])?; + validate_field_metadata(&fields[1])?; + Kind::Map(Box::new(pb::Map { + key_type: Some(Box::new(spark_type(fields[0].data_type())?)), + value_type: Some(Box::new(spark_type(fields[1].data_type())?)), + value_contains_null: fields[1].is_nullable(), + ..Default::default() + })) + } + other => { + return Err(Status::unimplemented(format!( + "Arrow result type {other:?}" + ))) + } + }; + Ok(proto::DataType { kind: Some(kind) }) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + fn annotated(name: &str, data_type: DataType, nullable: bool) -> Field { + Field::new(name, data_type, nullable).with_metadata(HashMap::from([( + "producer.note".into(), + "cannot be represented in Spark schemas".into(), + )])) + } + + #[test] + fn declared_metadata_is_rejected_on_list_and_every_map_child() { + let entries = |key: Field, value: Field| { + Field::new("entries", DataType::Struct(vec![key, value].into()), false) + }; + let types = [ + DataType::List(Arc::new(annotated("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(annotated("item", DataType::Int64, true))), + DataType::Map( + Arc::new(annotated( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + ] + .into(), + ), + false, + )), + false, + ), + DataType::Map( + Arc::new(entries( + annotated("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + )), + false, + ), + DataType::Map( + Arc::new(entries( + Field::new("key", DataType::Utf8, false), + annotated("value", DataType::Int64, true), + )), + false, + ), + ]; + for data_type in types { + let schema = Schema::new(vec![Field::new("nested", data_type, true)]); + assert_eq!( + result_schema(&schema).unwrap_err().code(), + tonic::Code::Unimplemented + ); + // Batch annotations remain acceptable when comparing the declared + // Spark-visible shape, including all nested child fields. + comparable_schema(&schema).unwrap(); + } + } +} diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs new file mode 100644 index 000000000..ce5ddd322 --- /dev/null +++ b/crates/spark-connect-server/src/service.rs @@ -0,0 +1,1205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::{HashMap, VecDeque}; +use std::panic::AssertUnwindSafe; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; + +use arrow::datatypes::{Schema, SchemaRef}; +use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; +use arrow::record_batch::RecordBatch; +use futures::{FutureExt, Stream, StreamExt}; +use prost::Message; +use spark_connect_proto as proto; +use tokio::sync::{mpsc, watch}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use uuid::Uuid; + +use crate::engine::{Engine, EngineSession, PreparedOperation, ResultBatch, SessionContext}; +use crate::grpc::spark_connect_service_server::SparkConnectService; +use crate::schema::{self, result_schema, MAX_METADATA_RESPONSE_BYTES}; + +type Output = Pin> + Send>>; +type SessionKey = (String, String); +const MAX_RELEASED_SESSIONS: usize = 4096; +const RELEASE_WAIT: std::time::Duration = std::time::Duration::from_secs(2); + +fn parse_uuid(value: &str, name: &str) -> Result { + crate::ids::parse_uuid(value) + .ok_or_else(|| Status::invalid_argument(format!("invalid UUID in {name}"))) +} + +async fn catch_engine_panic( + action: impl std::future::Future> + Send, +) -> Result { + AssertUnwindSafe(action) + .catch_unwind() + .await + .unwrap_or_else(|_| Err(Status::internal("engine request panicked"))) +} + +struct Session { + id: String, + backend: Arc, + cancelled: watch::Sender, + operations: Mutex>>, +} + +struct Operation { + cancelled: watch::Sender, + done: watch::Sender, +} + +impl Session { + fn new(backend: Arc) -> Self { + Self { + id: Uuid::new_v4().to_string(), + backend, + cancelled: watch::channel(false).0, + operations: Mutex::new(HashMap::new()), + } + } + + async fn while_open( + &self, + action: impl std::future::Future> + Send, + ) -> Result { + let mut cancelled = self.cancelled.subscribe(); + if *cancelled.borrow() { + return Err(Status::cancelled("session released")); + } + tokio::select! { + biased; + _ = cancelled.changed() => Err(Status::cancelled("session released")), + result = action => result, + } + } + + fn operation( + self: &Arc, + id: &str, + ) -> Result<(Arc, watch::Receiver), Status> { + // Subscribe before publishing: ReleaseAll can cancel as soon as the + // operation appears in the registry, before preparation is first polled. + let (cancelled, cancellation) = watch::channel(false); + let running = Arc::new(Operation { + cancelled, + done: watch::channel(false).0, + }); + let mut operations = self + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if operations.contains_key(id) { + return Err(Status::already_exists("operation is already running")); + } + operations.insert(id.to_owned(), running.clone()); + Ok(( + Arc::new(OperationGuard { + session: self.clone(), + id: id.to_owned(), + running, + }), + cancellation, + )) + } +} + +// This guard spans preparation and execution, including eager commands. Drop the +// backend future/stream before the guard so ReleaseAll observes completed teardown. +struct OperationGuard { + session: Arc, + id: String, + running: Arc, +} + +impl Drop for OperationGuard { + fn drop(&mut self) { + self.session + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.id); + self.running.done.send_replace(true); + } +} + +#[derive(Default)] +struct SessionRegistry { + // None is a recently released-session tombstone, retained until eviction. + entries: HashMap>>, + // Remember the most recent distinct closed IDs in release order. Repeated + // releases do not refresh retention. After eviction, an old ID may create a + // fresh session; active sessions never count toward this finite budget. + released: VecDeque, +} + +impl SessionRegistry { + fn release(&mut self, key: SessionKey, allow_reconnect: bool) -> Option> { + // Releasing a missing or already closed session is a no-op. In + // particular, a later reconnecting release cannot erase a tombstone. + let session = self.entries.get_mut(&key)?.take()?; + if allow_reconnect { + self.entries.remove(&key); + } else { + self.released.push_back(key); + if self.released.len() > MAX_RELEASED_SESSIONS { + if let Some(oldest) = self.released.pop_front() { + self.entries.remove(&oldest); + } + } + } + Some(session) + } +} + +pub(crate) struct ConnectService { + engine: Arc, + options: crate::ServerOptions, + sessions: Mutex, + closing: std::sync::atomic::AtomicBool, +} + +impl ConnectService { + pub(crate) fn new(engine: Arc, options: crate::ServerOptions) -> Self { + Self { + engine, + options, + sessions: Mutex::new(SessionRegistry::default()), + closing: std::sync::atomic::AtomicBool::new(false), + } + } + + fn key(id: &str, user: Option<&proto::UserContext>) -> SessionKey { + ( + user.map(|u| u.user_id.clone()).unwrap_or_default(), + id.into(), + ) + } + + fn session( + &self, + id: &str, + user: Option<&proto::UserContext>, + observed: Option<&str>, + ) -> Result, Status> { + self.find_session(id, user, observed, true) + } + + fn find_session( + &self, + id: &str, + user: Option<&proto::UserContext>, + observed: Option<&str>, + create: bool, + ) -> Result, Status> { + if create { + parse_uuid(id, "session_id")?; + } + let key = Self::key(id, user); + let mut sessions = self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.closing.load(std::sync::atomic::Ordering::SeqCst) { + return Err(Status::unavailable("server is shutting down")); + } + let session = match sessions.entries.get(&key) { + Some(Some(session)) => session.clone(), + Some(None) => return Err(Status::failed_precondition("session was released")), + None if !create => return Err(Status::not_found("session does not exist")), + // An observed ID refers to a previous incarnation; a new session + // cannot satisfy it. Reject before allocating backend resources. + None if observed.is_some() => { + return Err(Status::failed_precondition( + "server-side session ID changed", + )); + } + None => { + let backend = self.engine.create_session(SessionContext { + session_id: id.to_owned(), + user_context: user.cloned(), + })?; + let session = Arc::new(Session::new(backend)); + sessions.entries.insert(key, Some(session.clone())); + session + } + }; + if observed.is_some_and(|id| id != session.id) { + return Err(Status::failed_precondition( + "server-side session ID changed", + )); + } + Ok(session) + } + + pub(crate) fn cancel_all(&self) { + self.closing + .store(true, std::sync::atomic::Ordering::SeqCst); + // Stalled HTTP/2 connections can retain the service after the bounded + // transport drain. Remove its registry ownership now so backend sessions + // can close once their cancelled operations finish. + let sessions = std::mem::take( + &mut *self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ); + for session in sessions.entries.into_values().flatten() { + session.cancelled.send_replace(true); + } + } +} + +fn response( + session_id: &str, + session: &Session, + operation_id: &str, + response_type: Option, +) -> proto::ExecutePlanResponse { + proto::ExecutePlanResponse { + session_id: session_id.into(), + server_side_session_id: session.id.clone(), + operation_id: operation_id.into(), + response_id: Uuid::new_v4().to_string(), + response_type, + ..Default::default() + } +} + +fn checked_response( + message: proto::ExecutePlanResponse, +) -> Result { + if message.encoded_len() > MAX_METADATA_RESPONSE_BYTES { + return Err(Status::resource_exhausted( + "schema/command response exceeds the 3 MiB limit", + )); + } + Ok(message) +} + +fn complete(id: &str, session: &Session, operation: &str) -> proto::ExecutePlanResponse { + response( + id, + session, + operation, + Some(proto::execute_plan_response::ResponseType::ResultComplete( + proto::execute_plan_response::ResultComplete {}, + )), + ) +} + +fn encode_batch(batch: &RecordBatch) -> Result<(Vec, SchemaRef), Status> { + use arrow::datatypes::DataType; + use arrow::record_batch::RecordBatchOptions; + + // Materialize view arrays, including nested children, into the large-offset + // string/binary representation used for encoded RecordBatch results. + let source_schema = batch.schema(); + let fields = schema::map_fields(source_schema.fields(), &|field, data_type| { + let data_type = match data_type { + DataType::Utf8View => DataType::LargeUtf8, + DataType::BinaryView => DataType::LargeBinary, + other => other, + }; + if data_type == *field.data_type() { + field.clone() + } else { + Arc::new(field.as_ref().clone().with_data_type(data_type)) + } + }); + let changed = fields + .iter() + .zip(source_schema.fields()) + .any(|(target, source)| target.data_type() != source.data_type()); + let materialized = if changed { + let columns = batch + .columns() + .iter() + .zip(&fields) + .map(|(column, field)| { + if column.data_type() == field.data_type() { + Ok(column.clone()) + } else { + arrow::compute::cast(column.as_ref(), field.data_type()) + } + }) + .collect::, _>>() + .map_err(|error| Status::internal(error.to_string()))?; + let target_schema = Arc::new(Schema::new_with_metadata( + fields, + source_schema.metadata().clone(), + )); + Some( + RecordBatch::try_new_with_options( + target_schema, + columns, + &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + ) + .map_err(|error| Status::internal(error.to_string()))?, + ) + } else { + None + }; + let batch = materialized.as_ref().unwrap_or(batch); + let mut bytes = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut bytes, batch.schema().as_ref()) + .map_err(|e| Status::internal(e.to_string()))?; + writer + .write(batch) + .map_err(|e| Status::internal(e.to_string()))?; + writer + .finish() + .map_err(|e| Status::internal(e.to_string()))?; + } + Ok((bytes, batch.schema())) +} + +// PySpark assembles one table from all IPC batches. Its physical types and +// nullability must agree across batches, while field/schema metadata may differ. +fn check_output_schema(schema: &Schema, expected: &mut Option) -> Result<(), Status> { + use arrow::datatypes::Field; + + let actual = Schema::new(schema::map_fields(schema.fields(), &|field, data_type| { + Arc::new(Field::new(field.name(), data_type, field.is_nullable())) + })); + match expected { + Some(expected) if *expected != actual => Err(Status::internal( + "Arrow output schema changed within one result", + )), + Some(_) => Ok(()), + None => { + *expected = Some(actual); + Ok(()) + } + } +} + +// Keep backend ownership until the batch is dropped, including on encoder panic. +struct EncodingBatch { + batch: RecordBatch, + _guard: Arc, +} + +impl EncodingBatch { + fn encode(self) -> Result<(Vec, SchemaRef), Status> { + encode_batch(&self.batch) + } +} + +fn arrow_chunk_size( + options: &[proto::execute_plan_request::RequestOption], + maximum: usize, +) -> Option { + use proto::execute_plan_request::request_option::RequestOption; + + let mut chunking = options + .iter() + .filter_map(|option| match &option.request_option { + Some(RequestOption::ResultChunkingOptions(options)) => Some(options), + _ => None, + }); + if maximum == 0 + || !chunking + .clone() + .any(|options| options.allow_arrow_batch_chunking) + { + return None; + } + Some( + chunking + .find_map(|options| options.preferred_arrow_chunk_size) + // Validate i64 directly; Scala's toInt can wrap an invalid preference. + .and_then(|bytes| usize::try_from(bytes).ok()) + .filter(|bytes| (1024..=maximum).contains(bytes)) + .unwrap_or(maximum), + ) +} + +async fn send_batch( + sender: &mpsc::Sender>, + id: &str, + operation: &str, + batch: ResultBatch, + batch_schema: &Schema, + offset: &mut i64, + output_schema: &mut Option, + guard: &Arc, + chunk_size: Option, +) -> Result { + // Reject unrepresentable counts/offsets before encoding or sending any chunk. + let row_count = match &batch { + ResultBatch::Arrow(batch) => i64::try_from(batch.num_rows()) + .map_err(|_| Status::resource_exhausted("result batch row count exceeds i64"))?, + ResultBatch::Ipc { row_count, .. } if *row_count < 0 => { + return Err(Status::internal("negative engine IPC row count")); + } + ResultBatch::Ipc { row_count, .. } => *row_count, + }; + let next_offset = offset + .checked_add(row_count) + .ok_or_else(|| Status::resource_exhausted("result row offset exceeds i64"))?; + let data: prost::bytes::Bytes = match batch { + ResultBatch::Arrow(batch) => { + // A blocking encoder cannot be cancelled once started. Retain engine + // ownership until its batch is dropped, even if the caller goes away. + let encoding = EncodingBatch { + batch, + _guard: guard.clone(), + }; + let (data, schema) = tokio::task::spawn_blocking(move || encoding.encode()) + .await + .map_err(|_| Status::internal("Arrow result encoding task failed"))??; + check_output_schema(schema.as_ref(), output_schema)?; + data.into() + } + ResultBatch::Ipc { data, .. } => { + check_output_schema(batch_schema, output_schema)?; + data.into() + } + }; + let mut payload = proto::execute_plan_response::ArrowBatch { + row_count, + start_offset: Some(*offset), + ..Default::default() + }; + let Some(chunk_bytes) = chunk_size else { + // As on the JVM, clients without chunking opt-in receive the original + // whole IPC batch. Their gRPC receive limit must accommodate it. + payload.data = data; + let message = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + payload, + )), + ); + *offset = next_offset; + return Ok(sender.send(Ok(message)).await.is_ok()); + }; + let chunks = data.len().div_ceil(chunk_bytes).max(1); + payload.num_chunks_in_batch = Some( + i64::try_from(chunks) + .map_err(|_| Status::resource_exhausted("Arrow chunk count exceeds i64"))?, + ); + for index in 0..chunks { + let from = index * chunk_bytes; + let to = from.saturating_add(chunk_bytes).min(data.len()); + // Bytes slices share the original IPC allocation, including compressed + // buffers. Clients concatenate the chunks before decoding Arrow. + payload.data = data.slice(from..to); + payload.chunk_index = Some(index as i64); + let message = response( + id, + &guard.session, + operation, + Some(proto::execute_plan_response::ResponseType::ArrowBatch( + payload.clone(), + )), + ); + if sender.send(Ok(message)).await.is_err() { + return Ok(false); + } + } + *offset = next_offset; + Ok(true) +} + +#[tonic::async_trait] +impl SparkConnectService for ConnectService { + type ExecutePlanStream = Output; + type ReattachExecuteStream = Output; + + async fn execute_plan( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let chunk_size = arrow_chunk_size(&req.request_options, self.options.max_arrow_chunk_size); + let reattachable = req.request_options.iter().any(|option| { + matches!( + &option.request_option, + Some(proto::execute_plan_request::request_option::RequestOption::ReattachOptions( + options + )) if options.reattachable + ) + }); + let operation = match req.operation_id { + Some(id) => parse_uuid(&id, "operation_id")?, + None => Uuid::new_v4(), + } + .to_string(); + let op = req + .plan + .and_then(|p| p.op_type) + .ok_or_else(|| Status::invalid_argument("missing plan"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let mut cancelled = session.cancelled.subscribe(); + if *cancelled.borrow() { + return Err(Status::cancelled("session released")); + } + // Reserve before calling the backend: preparing a command can mutate state. + let (guard, mut operation_cancelled) = session.operation(&operation)?; + let prepared = tokio::select! { + biased; + _ = cancelled.changed() => return Err(Status::cancelled("session released")), + _ = operation_cancelled.changed() => return Err(Status::cancelled("operation released")), + prepared = catch_engine_panic(async { session.backend.prepare(op).await }) => prepared?, + }; + let query = match prepared { + PreparedOperation::Query(query) => query, + PreparedOperation::Command(payloads) => { + let mut messages = payloads + .into_iter() + .map(|payload| { + let message = + response(&req.session_id, &session, &operation, Some(payload)); + checked_response(message) + }) + .collect::, Status>>()?; + if reattachable { + messages.push(complete(&req.session_id, &session, &operation)); + } + return Ok(Response::new(Box::pin(tokio_stream::iter( + messages.into_iter().map(Ok), + )))); + } + }; + let arrow_schema = query.schema(); + let spark_schema = result_schema(&arrow_schema)?; + let expected_batch_schema = schema::comparable_schema(&arrow_schema)?; + let (sender, receiver) = mpsc::channel(2); + tokio::spawn(async move { + let run = async { + let mut header = response(&req.session_id, &session, &operation, None); + header.schema = Some(spark_schema); + let header = checked_response(header)?; + if sender.send(Ok(header)).await.is_err() { + return Ok(()); + } + let mut batches = query.execute().await?; + let mut offset = 0; + let mut output_schema = None; + let mut sent = false; + while let Some(batch) = batches.next().await { + let batch = batch?; + let batch_schema = match &batch { + ResultBatch::Arrow(batch) => batch.schema(), + ResultBatch::Ipc { data, .. } => { + StreamReader::try_new(data.as_slice(), None) + .map_err(|e| Status::internal(format!("invalid engine IPC: {e}")))? + .schema() + } + }; + let actual_schema = + schema::comparable_schema(batch_schema.as_ref()).map_err(|error| { + Status::internal(format!( + "invalid engine batch schema: {}", + error.message() + )) + })?; + if actual_schema != expected_batch_schema { + return Err(Status::internal( + "engine batch schema differs from its declared schema", + )); + } + if !send_batch( + &sender, + &req.session_id, + &operation, + batch, + batch_schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + chunk_size, + ) + .await? + { + return Ok(()); + } + sent = true; + } + // Teardown can itself panic. Complete only after the backend + // stream has been dropped within the panic boundary. + drop(batches); + if !sent { + let batch = RecordBatch::new_empty(arrow_schema); + let batch_schema = batch.schema(); + if !send_batch( + &sender, + &req.session_id, + &operation, + batch.into(), + batch_schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + chunk_size, + ) + .await? + { + return Ok(()); + } + } + if reattachable { + let _ = sender + .send(Ok(complete(&req.session_id, &session, &operation))) + .await; + } + Ok::<_, Status>(()) + }; + let result = AssertUnwindSafe(async { + tokio::select! { + biased; + _ = cancelled.changed() => Err(Status::cancelled("session released")), + _ = operation_cancelled.changed() => Err(Status::cancelled("operation released")), + _ = sender.closed() => Ok(()), + result = run => result, + } + }) + .catch_unwind() + .await + .unwrap_or_else(|_| Err(Status::internal("engine execution panicked"))); + drop(guard); + // A slow peer can keep this terminal send pending after cancellation. + // Backend lifetime must end before waiting for that response slot. + drop(session); + if let Err(error) = result { + let _ = sender.send(Err(error)).await; + } + }); + Ok(Response::new(Box::pin(ReceiverStream::new(receiver)))) + } + + async fn analyze_plan( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let input = req + .analyze + .ok_or_else(|| Status::invalid_argument("missing analysis"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let result = session + .while_open(catch_engine_panic(async { + session.backend.analyze(input).await + })) + .await?; + if let proto::analyze_plan_response::Result::Schema(result) = &result { + if let Some(schema) = &result.schema { + schema::validate_schema_size(schema)?; + } + } + Ok(Response::new(proto::AnalyzePlanResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + result: Some(result), + })) + } + + async fn config( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let input = req + .operation + .and_then(|operation| operation.op_type) + .ok_or_else(|| Status::invalid_argument("missing config operation"))?; + let session = self.session( + &req.session_id, + req.user_context.as_ref(), + req.client_observed_server_side_session_id.as_deref(), + )?; + let result = session + .while_open(catch_engine_panic(async { + session.backend.config(input).await + })) + .await?; + Ok(Response::new(proto::ConfigResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + pairs: result.pairs, + warnings: result.warnings, + })) + } + + async fn release_execute( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + // Spark ignores the observed incarnation here and looks up the supplied + // operation string as-is, including IDs that do not parse as UUIDs. + let session = self.find_session(&req.session_id, req.user_context.as_ref(), None, false)?; + // Completed responses are not retained; release is idempotent for absent operations. + let running = session + .operations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&req.operation_id) + .cloned(); + let operation_id = running.as_ref().map(|_| req.operation_id.clone()); + if matches!( + req.release, + Some(proto::release_execute_request::Release::ReleaseAll(_)) + ) { + if let Some(running) = running { + let mut done = running.done.subscribe(); + running.cancelled.send_replace(true); + tokio::time::timeout(RELEASE_WAIT, async { + while !*done.borrow_and_update() { + if done.changed().await.is_err() { + break; + } + } + }) + .await + .map_err(|_| { + Status::deadline_exceeded("operation cancellation is still draining") + })?; + } + } + Ok(Response::new(proto::ReleaseExecuteResponse { + session_id: req.session_id, + server_side_session_id: session.id.clone(), + operation_id, + })) + } + + async fn release_session( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let key = Self::key(&req.session_id, req.user_context.as_ref()); + let session = { + let mut sessions = self + .sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + sessions.release(key, req.allow_reconnect) + }; + let server_id = session.as_ref().map(|s| s.id.clone()).unwrap_or_default(); + if let Some(session) = session { + session.cancelled.send_replace(true); + } + Ok(Response::new(proto::ReleaseSessionResponse { + session_id: req.session_id, + server_side_session_id: server_id, + })) + } + + async fn reattach_execute( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "execution reattachment is not supported", + )) + } + async fn add_artifacts( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("artifact uploads are not supported")) + } + async fn artifact_status( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("artifact status is not supported")) + } + async fn interrupt( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "operation interruption is not supported", + )) + } + async fn fetch_error_details( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "enriched error details are not supported", + )) + } + async fn clone_session( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "server-side session cloning is not supported", + )) + } + async fn get_status( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("status queries are not supported")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const CHUNK_SIZE: usize = 1024 * 1024; + + struct RegistryEngine; + struct RegistrySession; + + impl Engine for RegistryEngine { + fn create_session( + &self, + _context: SessionContext, + ) -> Result, Status> { + Ok(Arc::new(RegistrySession)) + } + } + + #[tonic::async_trait] + impl EngineSession for RegistrySession { + async fn prepare( + &self, + _operation: proto::plan::OpType, + ) -> Result { + Err(Status::unimplemented( + "registry fixture does not execute queries", + )) + } + + async fn analyze( + &self, + _input: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented( + "registry fixture does not analyze queries", + )) + } + + async fn config( + &self, + _input: proto::config_request::operation::OpType, + ) -> Result { + Err(Status::unimplemented( + "registry fixture has no configuration", + )) + } + } + + async fn release(service: &ConnectService, id: &str, allow_reconnect: bool) { + service + .release_session(Request::new(proto::ReleaseSessionRequest { + session_id: id.into(), + allow_reconnect, + ..Default::default() + })) + .await + .unwrap(); + } + + #[test] + fn schema_response_limit_includes_protocol_envelope() { + let session = Session::new(Arc::new(RegistrySession)); + let schema = proto::DataType { + kind: Some(proto::data_type::Kind::Struct(proto::data_type::Struct { + fields: vec![proto::data_type::StructField { + name: "x".repeat(MAX_METADATA_RESPONSE_BYTES - 64), + ..Default::default() + }], + ..Default::default() + })), + }; + assert!(schema.encoded_len() < MAX_METADATA_RESPONSE_BYTES); + let mut header = response( + &Uuid::new_v4().to_string(), + &session, + &Uuid::new_v4().to_string(), + None, + ); + header.schema = Some(schema); + assert_eq!( + checked_response(header).unwrap_err().code(), + tonic::Code::ResourceExhausted + ); + } + + #[cfg(target_pointer_width = "64")] + #[tokio::test] + async fn unrepresentable_arrow_row_counts_fail_before_encoding() { + use arrow::array::NullArray; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatchOptions; + + let rows = i64::MAX as usize + 1; + let empty = RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap(); + let nulls = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), + vec![Arc::new(NullArray::new(rows))], + ) + .unwrap(); + for batch in [empty, nulls] { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("row_count").unwrap(); + let (sender, mut receiver) = mpsc::channel(2); + let mut offset = 7; + let mut output_schema = None; + let error = send_batch( + &sender, + "session", + "row_count", + batch.clone().into(), + batch.schema().as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(CHUNK_SIZE), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert_eq!(error.message(), "result batch row count exceeds i64"); + assert_eq!(offset, 7); + assert!(output_schema.is_none()); + assert!(receiver.try_recv().is_err()); + } + } + + #[tokio::test] + async fn overflowing_engine_batches_do_not_partially_emit() { + use arrow::array::LargeStringArray; + use arrow::datatypes::{DataType, Field, Schema}; + + let schema = Arc::new(Schema::new(vec![Field::new( + "text", + DataType::LargeUtf8, + false, + )])); + let small = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(LargeStringArray::from(vec![""]))], + ) + .unwrap(); + let large = "z".repeat(2 * 1024 * 1024); + let oversized = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(LargeStringArray::from(vec![large.as_str(); 2]))], + ) + .unwrap(); + for ipc in [false, true] { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("row_offset").unwrap(); + let (sender, mut receiver) = mpsc::channel(4); + let mut offset = i64::MAX - 2; + let mut output_schema = None; + send_batch( + &sender, + "session", + "row_offset", + small.clone().into(), + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(CHUNK_SIZE), + ) + .await + .unwrap(); + let batch = if ipc { + let (data, _) = encode_batch(&oversized).unwrap(); + assert!(data.len() > CHUNK_SIZE); + ResultBatch::Ipc { data, row_count: 2 } + } else { + oversized.clone().into() + }; + let error = send_batch( + &sender, + "session", + "row_offset", + batch, + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(CHUNK_SIZE), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + assert_eq!(error.message(), "result row offset exceeds i64"); + assert_eq!(offset, i64::MAX - 1); + let previous = receiver.try_recv().unwrap().unwrap(); + let Some(proto::execute_plan_response::ResponseType::ArrowBatch(previous)) = + previous.response_type + else { + panic!("expected the preceding valid Arrow response"); + }; + assert_eq!(previous.start_offset, Some(i64::MAX - 2)); + assert_eq!(previous.row_count, 1); + assert!(receiver.try_recv().is_err()); + // A one-row batch still fits exactly at the signed boundary. + send_batch( + &sender, + "session", + "row_offset", + small.clone().into(), + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(CHUNK_SIZE), + ) + .await + .unwrap(); + assert_eq!(offset, i64::MAX); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn cancelled_encoding_wait_retains_operation_until_blocking_task_finishes() { + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let operation = Uuid::new_v4().to_string(); + let (guard, _) = session.operation(&operation).unwrap(); + let mut done = guard.running.done.subscribe(); + let encoding = EncodingBatch { + batch: RecordBatch::new_empty(Arc::new(arrow::datatypes::Schema::empty())), + _guard: guard.clone(), + }; + let (started, ready) = tokio::sync::oneshot::channel(); + let (release, resume) = std::sync::mpsc::channel(); + let waiting = tokio::spawn(async move { + tokio::task::spawn_blocking(move || { + let _ = started.send(()); + resume + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + encoding.encode() + }) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(5), ready) + .await + .unwrap() + .unwrap(); + waiting.abort(); + let _ = waiting.await; + drop(guard); + assert!(!*done.borrow()); + assert!(session.operations.lock().unwrap().contains_key(&operation)); + release.send(()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while !*done.borrow_and_update() { + done.changed().await.unwrap(); + } + }) + .await + .unwrap(); + assert!(!session.operations.lock().unwrap().contains_key(&operation)); + } + + #[tokio::test] + async fn released_session_retention_is_bounded_without_evicting_active_sessions() { + let service = + ConnectService::new(Arc::new(RegistryEngine), crate::ServerOptions::default()); + let active_id = Uuid::new_v4().to_string(); + let active = service.session(&active_id, None, None).unwrap(); + + let oldest = Uuid::new_v4().to_string(); + let previous = service.session(&oldest, None, None).unwrap(); + release(&service, &oldest, false).await; + assert!(*previous.cancelled.borrow()); + release(&service, &oldest, false).await; + release(&service, &oldest, true).await; + assert!(matches!(service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition)); + { + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.entries.len(), 2); + assert_eq!(registry.released.len(), 1); + } + let mut newest = oldest.clone(); + for _ in 1..MAX_RELEASED_SESSIONS { + newest = Uuid::new_v4().to_string(); + service.session(&newest, None, None).unwrap(); + release(&service, &newest, false).await; + } + release(&service, &oldest, false).await; + assert!(matches!( + service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition + )); + { + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.released.len(), MAX_RELEASED_SESSIONS); + assert_eq!(registry.entries.len(), MAX_RELEASED_SESSIONS + 1); + } + + // Unknown releases neither allocate tombstones nor evict closed IDs. + let unknown = Uuid::new_v4().to_string(); + release(&service, &unknown, false).await; + release(&service, &unknown, true).await; + assert!(matches!(service.session(&oldest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition)); + service.session(&unknown, None, None).unwrap(); + release(&service, &unknown, false).await; + assert!(service.session(&oldest, None, None).is_ok()); + assert!(matches!( + service.session(&newest, None, None), + Err(status) if status.code() == tonic::Code::FailedPrecondition + )); + assert!(Arc::ptr_eq( + &active, + &service.session(&active_id, None, None).unwrap() + )); + let registry = service.sessions.lock().unwrap(); + assert_eq!(registry.released.len(), MAX_RELEASED_SESSIONS); + assert_eq!(registry.entries.len(), MAX_RELEASED_SESSIONS + 2); + } + + #[tokio::test] + async fn reconnecting_release_of_live_session_allows_a_new_backend() { + let service = + ConnectService::new(Arc::new(RegistryEngine), crate::ServerOptions::default()); + let id = Uuid::new_v4().to_string(); + let previous = service.session(&id, None, None).unwrap(); + release(&service, &id, true).await; + assert!(*previous.cancelled.borrow()); + let current = service.session(&id, None, None).unwrap(); + assert!(!Arc::ptr_eq(¤t, &previous)); + } +} diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs new file mode 100644 index 000000000..a7dd12657 --- /dev/null +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -0,0 +1,2203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Real gRPC tests of the injected backend boundary. The fake is independent of Spark execution. + +#[path = "support/arrow.rs"] +mod arrow_support; +#[path = "support/requests.rs"] +mod requests; +#[path = "support/server.rs"] +mod server; + +use std::collections::HashMap; +use std::future::Future; +use std::io::Cursor; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::task::{Context, Poll}; +use std::time::Duration; + +use arrow::array::{Int64Array, StringArray}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::ipc::reader::StreamReader; +use arrow::ipc::writer::StreamWriter; +use arrow::record_batch::RecordBatch; +use arrow_support::chunking; +use futures::{Stream, StreamExt}; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use proto::spark_connect_service_client::SparkConnectServiceClient; +use requests::{ + assert_cancelled, config_request, reattachable_request, relation, release, release_session, + request, user, +}; +use server::Server; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::{Code, Status}; +use uuid::Uuid; + +const LIVE: i64 = 1_000_000; +const BLOCKED_PREPARE: i64 = 2_000_000; +const MISMATCHED_BATCH: i64 = 3_000_000; +const ANNOTATED_BATCH: i64 = 4_000_000; +const FLOODED_RESULT: i64 = 5_000_000; +const PANICKING_EXECUTION: i64 = 6_000_000; +const PANICKING_STREAM: i64 = 7_000_000; +const PANICKING_STREAM_DROP: i64 = 8_000_000; +const MISMATCHED_IPC_BATCH: i64 = 9_000_000; +const INVALID_IPC_BATCH: i64 = 10_000_000; +const CHUNKED_RESULT: i64 = 11_000_000; + +#[derive(Default)] +struct Spy { + factory_calls: Mutex>, + fail_next: AtomicBool, + sessions_dropped: AtomicUsize, + prepares: AtomicUsize, + commands: AtomicUsize, + analyses: AtomicUsize, + configs: AtomicUsize, + executions: AtomicUsize, + streams_dropped: AtomicUsize, + batches_polled: AtomicUsize, + blocked_prepares_dropped: AtomicUsize, + blocked_analyses_dropped: AtomicUsize, + blocked_configs_dropped: AtomicUsize, + metadata_drop_observations: Mutex>, + slow_drop: Mutex)>>, + stream_drop_gate: Mutex>>, + request_panic: Mutex>, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Call { + Prepare, + Analyze, + Config, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum PanicAt { + Construction, + Poll, +} + +impl Spy { + fn panic_once(&self, call: Call) -> Option { + let mut next = self.request_panic.lock().unwrap(); + if next.as_ref().is_some_and(|(target, _)| *target == call) { + next.take().map(|(_, at)| at) + } else { + None + } + } +} + +type EngineFuture<'a, T> = Pin> + Send + 'a>>; + +#[derive(Default)] +struct DropGate { + released: Mutex, + condition: Condvar, + entered: AtomicUsize, + exited: AtomicBool, + timed_out: AtomicBool, +} + +impl DropGate { + fn wait(&self) { + let released = self.released.lock().unwrap(); + self.entered.fetch_add(1, Ordering::SeqCst); + let (released, timeout) = self + .condition + .wait_timeout_while(released, Duration::from_secs(10), |released| !*released) + .unwrap(); + self.timed_out + .store(timeout.timed_out() && !*released, Ordering::SeqCst); + self.exited.store(true, Ordering::SeqCst); + } + + fn release(&self) { + *self.released.lock().unwrap() = true; + self.condition.notify_all(); + } +} + +// Even a failing assertion must never leave a worker blocked in the fake Drop. +struct UnblockDrop(Arc); + +impl Drop for UnblockDrop { + fn drop(&mut self) { + self.0.release(); + } +} + +struct FakeEngine(Arc); + +impl Engine for FakeEngine { + fn create_session(&self, context: SessionContext) -> Result, Status> { + let drop_gate = self + .0 + .slow_drop + .lock() + .unwrap() + .as_ref() + .filter(|(id, _)| id == &context.session_id) + .map(|(_, gate)| gate.clone()); + let user = context + .user_context + .map(|user| user.user_id) + .unwrap_or_default(); + self.0 + .factory_calls + .lock() + .unwrap() + .push((user, context.session_id)); + if self.0.fail_next.swap(false, Ordering::SeqCst) { + return Err(Status::unavailable( + "fake factory is temporarily unavailable", + )); + } + Ok(Arc::new(FakeSession { + spy: self.0.clone(), + configs: Mutex::new(HashMap::new()), + views: Mutex::new(HashMap::new()), + drop_gate, + })) + } +} + +struct FakeSession { + spy: Arc, + configs: Mutex>, + views: Mutex>, + drop_gate: Option>, +} + +impl Drop for FakeSession { + fn drop(&mut self) { + if let Some(gate) = &self.drop_gate { + // Keep Drop synchronous, while allowing Tokio to replace this worker + // so the test measures registry contention rather than timer starvation. + tokio::task::block_in_place(|| gate.wait()); + } + self.spy.metadata_drop_observations.lock().unwrap().push(( + self.spy.blocked_analyses_dropped.load(Ordering::SeqCst), + self.spy.blocked_configs_dropped.load(Ordering::SeqCst), + )); + self.spy.sessions_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +struct BlockedPrepare(Arc); + +impl Drop for BlockedPrepare { + fn drop(&mut self) { + self.0 + .blocked_prepares_dropped + .fetch_add(1, Ordering::SeqCst); + } +} + +enum MetadataCall { + Analysis, + Config, +} + +struct BlockedMetadata { + spy: Arc, + call: MetadataCall, +} + +impl Drop for BlockedMetadata { + fn drop(&mut self) { + let counter = match self.call { + MetadataCall::Analysis => &self.spy.blocked_analyses_dropped, + MetadataCall::Config => &self.spy.blocked_configs_dropped, + }; + counter.fetch_add(1, Ordering::SeqCst); + } +} + +// Explicit async-trait signatures also exercise a panic before a future is returned. +impl EngineSession for FakeSession { + fn prepare<'life0, 'async_trait>( + &'life0 self, + op: proto::plan::OpType, + ) -> EngineFuture<'async_trait, PreparedOperation> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Prepare); + assert!(panic != Some(PanicAt::Construction), "prepare construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "prepare poll"); + self.spy.prepares.fetch_add(1, Ordering::SeqCst); + let (value, live, batch_mode) = match op { + proto::plan::OpType::Root(relation) => match relation.rel_type { + Some(proto::relation::RelType::Range(range)) => { + if range.end == BLOCKED_PREPARE { + let _dropped = BlockedPrepare(self.spy.clone()); + std::future::pending::<()>().await; + } + if range.end == -1 { + return Err(Status::invalid_argument("fake rejected the plan")); + } + // Deliberately unlike range semantics: this proves the injected + // backend, rather than the default planner, executed the request. + let batch_mode = match range.end { + MISMATCHED_BATCH => BatchMode::WrongType, + MISMATCHED_IPC_BATCH => BatchMode::WrongTypeIpc, + INVALID_IPC_BATCH => BatchMode::InvalidIpc, + ANNOTATED_BATCH => BatchMode::DifferentAnnotations, + FLOODED_RESULT => BatchMode::Flooded, + CHUNKED_RESULT => BatchMode::Chunked, + PANICKING_EXECUTION => BatchMode::PanicExecute, + PANICKING_STREAM => BatchMode::PanicStream, + PANICKING_STREAM_DROP => BatchMode::PanicDrop, + _ => BatchMode::Declared, + }; + (999, range.end == LIVE, batch_mode) + } + Some(proto::relation::RelType::Read(read)) => { + let Some(proto::read::ReadType::NamedTable(table)) = read.read_type else { + return Err(Status::unimplemented("fake read")); + }; + let value = self + .views + .lock() + .unwrap() + .get(&table.unparsed_identifier) + .copied() + .ok_or_else(|| Status::not_found("fake view not found"))?; + (value, false, BatchMode::Declared) + } + _ => return Err(Status::unimplemented("fake relation")), + }, + proto::plan::OpType::Command(command) => { + self.spy.commands.fetch_add(1, Ordering::SeqCst); + match command.command_type { + Some(proto::command::CommandType::CreateDataframeView(view)) => { + let Some(proto::relation::RelType::Range(input)) = + view.input.and_then(|input| input.rel_type) + else { + return Err(Status::invalid_argument( + "fake view requires range input", + )); + }; + let mut views = self.views.lock().unwrap(); + if !view.replace && views.contains_key(&view.name) { + return Err(Status::already_exists("fake view already exists")); + } + views.insert(view.name, input.end); + return Ok(PreparedOperation::Command(Vec::new())); + } + Some(proto::command::CommandType::SqlCommand(command)) => { + return Ok(PreparedOperation::Command(vec![ + ResponseType::SqlCommandResult( + proto::execute_plan_response::SqlCommandResult { + relation: command.input, + }, + ), + ])); + } + _ => return Err(Status::unimplemented("fake command")), + } + } + _ => return Err(Status::unimplemented("fake operation")), + }; + Ok(PreparedOperation::Query(Box::new(FakeQuery { + spy: self.spy.clone(), + value, + live, + batch_mode, + }))) + }) + } + + fn analyze<'life0, 'async_trait>( + &'life0 self, + input: proto::analyze_plan_request::Analyze, + ) -> EngineFuture<'async_trait, proto::analyze_plan_response::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Analyze); + assert!(panic != Some(PanicAt::Construction), "analyze construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "analyze poll"); + self.spy.analyses.fetch_add(1, Ordering::SeqCst); + match input { + proto::analyze_plan_request::Analyze::Schema(_) => { + Ok(proto::analyze_plan_response::Result::Schema( + proto::analyze_plan_response::Schema { + schema: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Struct( + proto::data_type::Struct { + fields: vec![proto::data_type::StructField { + name: "x".repeat(3 * 1024 * 1024 + 1), + data_type: Some(proto::DataType { + kind: Some(proto::data_type::Kind::Long( + proto::data_type::Long::default(), + )), + }), + nullable: false, + metadata: None, + }], + ..Default::default() + }, + )), + }), + }, + )) + } + proto::analyze_plan_request::Analyze::IsStreaming(_) => { + let _dropped = BlockedMetadata { + spy: self.spy.clone(), + call: MetadataCall::Analysis, + }; + std::future::pending().await + } + proto::analyze_plan_request::Analyze::IsLocal(_) => { + Ok(proto::analyze_plan_response::Result::IsLocal( + proto::analyze_plan_response::IsLocal { is_local: true }, + )) + } + _ => Err(Status::unimplemented("fake analysis")), + } + }) + } + + fn config<'life0, 'async_trait>( + &'life0 self, + input: proto::config_request::operation::OpType, + ) -> EngineFuture<'async_trait, ConfigResult> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let panic = self.spy.panic_once(Call::Config); + assert!(panic != Some(PanicAt::Construction), "config construction"); + Box::pin(async move { + assert!(panic != Some(PanicAt::Poll), "config poll"); + use proto::config_request::operation::OpType; + self.spy.configs.fetch_add(1, Ordering::SeqCst); + if matches!(&input, OpType::Get(get) if get.keys.iter().any(|key| key == "fake.block")) + { + let _dropped = BlockedMetadata { + spy: self.spy.clone(), + call: MetadataCall::Config, + }; + return std::future::pending().await; + } + let mut configs = self.configs.lock().unwrap(); + match input { + OpType::Set(set) => { + for pair in set.pairs { + configs.insert( + pair.key, + pair.value + .ok_or_else(|| Status::invalid_argument("fake config value"))?, + ); + } + Ok(ConfigResult { + pairs: Vec::new(), + warnings: vec!["handled by fake backend".into()], + }) + } + OpType::GetOption(get) => Ok(ConfigResult { + pairs: get + .keys + .into_iter() + .map(|key| { + let value = configs.get(&key).cloned(); + proto::KeyValue { key, value } + }) + .collect(), + warnings: Vec::new(), + }), + _ => Err(Status::unimplemented("fake config operation")), + } + }) + } +} + +struct FakeQuery { + spy: Arc, + value: i64, + live: bool, + batch_mode: BatchMode, +} + +enum BatchMode { + Declared, + WrongType, + WrongTypeIpc, + InvalidIpc, + DifferentAnnotations, + Flooded, + Chunked, + PanicExecute, + PanicStream, + PanicDrop, +} + +#[tonic::async_trait] +impl PreparedQuery for FakeQuery { + fn schema(&self) -> SchemaRef { + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Int64, + false, + )])) + } + + async fn execute(self: Box) -> Result { + self.spy.executions.fetch_add(1, Ordering::SeqCst); + if matches!(self.batch_mode, BatchMode::PanicExecute) { + panic!("fake query execution panic"); + } + let batch = match self.batch_mode { + BatchMode::Declared + | BatchMode::InvalidIpc + | BatchMode::PanicExecute + | BatchMode::PanicStream + | BatchMode::PanicDrop => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value]))], + ), + BatchMode::WrongType | BatchMode::WrongTypeIpc => RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Utf8, + false, + )])), + vec![Arc::new(StringArray::from(vec!["wrong type"]))], + ), + BatchMode::DifferentAnnotations => RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "backend_value", + DataType::Int64, + true, + ) + .with_metadata(HashMap::from([( + "backend.note".into(), + "ignored annotation".into(), + )]))])), + vec![Arc::new(Int64Array::from(vec![self.value]))], + ), + BatchMode::Chunked => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value; 512 * 1024]))], + ), + BatchMode::Flooded => RecordBatch::try_new( + self.schema(), + vec![Arc::new(Int64Array::from(vec![self.value; 64 * 1024]))], + ), + } + .unwrap(); + if matches!(self.batch_mode, BatchMode::Flooded) { + let spy = self.spy.clone(); + return Ok(Box::pin(SpyStream { + inner: Box::pin(futures::stream::repeat_with(move || { + spy.batches_polled.fetch_add(1, Ordering::SeqCst); + Ok(batch.clone().into()) + })), + spy: self.spy.clone(), + panic_on_drop: false, + })); + } + let result = match self.batch_mode { + BatchMode::WrongTypeIpc => { + let mut data = Vec::new(); + { + let mut writer = + StreamWriter::try_new(&mut data, batch.schema().as_ref()).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + } + ResultBatch::Ipc { data, row_count: 1 } + } + BatchMode::InvalidIpc => ResultBatch::Ipc { + data: b"invalid IPC".to_vec(), + row_count: 1, + }, + _ => batch.into(), + }; + let first = futures::stream::iter(vec![Ok(result)]); + let inner: BatchStream = if matches!(self.batch_mode, BatchMode::PanicStream) { + Box::pin(first.chain(futures::stream::poll_fn(|_| { + panic!("fake stream panic after a partial result"); + }))) + } else if self.live { + Box::pin(first.chain(futures::stream::pending())) + } else { + Box::pin(first) + }; + Ok(Box::pin(SpyStream { + inner, + spy: self.spy.clone(), + panic_on_drop: matches!(self.batch_mode, BatchMode::PanicDrop), + })) + } +} + +struct SpyStream { + inner: BatchStream, + spy: Arc, + panic_on_drop: bool, +} + +impl Stream for SpyStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + self.get_mut().inner.as_mut().poll_next(context) + } +} + +impl Drop for SpyStream { + fn drop(&mut self) { + let gate = self.spy.stream_drop_gate.lock().unwrap().clone(); + if let Some(gate) = gate { + tokio::task::block_in_place(|| gate.wait()); + } + self.spy.streams_dropped.fetch_add(1, Ordering::SeqCst); + assert!(!self.panic_on_drop, "fake stream teardown panic"); + } +} + +async fn start_server(spy: Arc) -> Server { + Server::start_with_engine(Arc::new(FakeEngine(spy))).await +} + +fn command_request( + session: &str, + command_type: proto::command::CommandType, +) -> proto::ExecutePlanRequest { + let mut request = request(session, 0); + request.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Command(proto::Command { + command_type: Some(command_type), + })), + }); + request +} + +fn get_config(session: &str) -> proto::ConfigRequest { + config_request( + session, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["spark.fake.setting".into()], + }), + ) +} + +async fn wait_for_count(counter: &AtomicUsize, expected: usize) { + tokio::time::timeout(Duration::from_secs(5), async { + while counter.load(Ordering::SeqCst) < expected { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("backend lifecycle event must arrive"); + assert_eq!(counter.load(Ordering::SeqCst), expected); +} + +async fn collect( + stream: &mut tonic::Streaming, +) -> Vec { + let mut responses = Vec::new(); + while let Some(response) = stream.message().await.unwrap() { + responses.push(response); + } + responses +} + +fn values(responses: &[proto::ExecutePlanResponse]) -> Vec { + let mut values = Vec::new(); + for response in responses { + if let Some(ResponseType::ArrowBatch(batch)) = &response.response_type { + for batch in StreamReader::try_new(Cursor::new(batch.data.clone()), None).unwrap() { + let batch = batch.unwrap(); + let array = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + values.extend((0..array.len()).map(|index| array.value(index))); + } + } + } + values +} + +async fn start_live( + server: &mut Server, + request: proto::ExecutePlanRequest, +) -> tonic::Streaming { + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + stream +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn injected_backend_owns_query_analysis_config_and_catalog_state() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut first = server.client.clone(); + let mut second = server.client.clone(); + let (left, right) = tokio::join!( + first.config(get_config(&session)), + second.config(get_config(&session)) + ); + left.unwrap(); + right.unwrap(); + assert_eq!( + spy.factory_calls.lock().unwrap().len(), + 1, + "concurrent requests must share one backend session" + ); + + let set = config_request( + &session, + proto::config_request::operation::OpType::Set(proto::config_request::Set { + pairs: vec![proto::KeyValue { + key: "spark.fake.setting".into(), + value: Some("backend-owned".into()), + }], + silent: Some(false), + }), + ); + let configured = server.client.config(set).await.unwrap().into_inner(); + assert_eq!(configured.warnings, vec!["handled by fake backend"]); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .pairs[0] + .value + .as_deref(), + Some("backend-owned") + ); + let analysis = server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsLocal( + proto::analyze_plan_request::IsLocal { + plan: request(&session, 99).plan, + }, + )), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + assert!(matches!( + analysis.result, + Some(proto::analyze_plan_response::Result::IsLocal( + proto::analyze_plan_response::IsLocal { is_local: true } + )) + )); + + let mut stream = server + .client + .execute_plan(reattachable_request(&session, 99)) + .await + .unwrap() + .into_inner(); + let responses = collect(&mut stream).await; + assert_eq!(values(&responses), vec![999]); + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + let create_view = command_request( + &session, + proto::command::CommandType::CreateDataframeView(proto::CreateDataFrameViewCommand { + input: Some(relation(77)), + name: "fake_view".into(), + is_global: false, + replace: false, + }), + ); + let mut created = server + .client + .execute_plan(create_view) + .await + .unwrap() + .into_inner(); + collect(&mut created).await; + let mut read_view = request(&session, 0); + read_view.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation { + rel_type: Some(proto::relation::RelType::Read(proto::Read { + read_type: Some(proto::read::ReadType::NamedTable(proto::read::NamedTable { + unparsed_identifier: "fake_view".into(), + ..Default::default() + })), + ..Default::default() + })), + ..Default::default() + })), + }); + let mut stream = server + .client + .execute_plan(read_view.clone()) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![77]); + let sql = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(13)), + ..Default::default() + }), + ); + let mut stream = server.client.execute_plan(sql).await.unwrap().into_inner(); + let responses = collect(&mut stream).await; + assert!(matches!( + responses[0].response_type, + Some(ResponseType::SqlCommandResult(_)) + )); + + let mut other_user = get_config(&session); + other_user.user_context = user("bob"); + assert_eq!( + server + .client + .config(other_user) + .await + .unwrap() + .into_inner() + .pairs[0] + .value, + None + ); + read_view.user_context = user("bob"); + assert_eq!( + server + .client + .execute_plan(read_view) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + let other_session = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .config(get_config(&other_session)) + .await + .unwrap() + .into_inner() + .pairs[0] + .value, + None + ); + assert_eq!( + *spy.factory_calls.lock().unwrap(), + vec![ + ("alice".into(), session.clone()), + ("bob".into(), session), + ("alice".into(), other_session), + ] + ); + assert_eq!(spy.analyses.load(Ordering::SeqCst), 1); + assert_eq!(spy.commands.load(Ordering::SeqCst), 2); + assert_eq!(spy.executions.load(Ordering::SeqCst), 2); + server.stop().await; + wait_for_count(&spy.sessions_dropped, 3).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invalid_requests_do_not_open_backends_and_factory_failures_can_retry() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .execute_plan(request("not-a-uuid", 0)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut invalid_operation = request(&session, 0); + invalid_operation.operation_id = Some("not-a-uuid".into()); + assert_eq!( + server + .client + .execute_plan(invalid_operation) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + for plan in [None, Some(proto::Plan { op_type: None })] { + let mut request = request(&session, 0); + request.plan = plan; + assert_eq!( + server + .client + .execute_plan(request) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + assert_eq!( + server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + for operation in [ + None, + Some(proto::config_request::Operation { op_type: None }), + ] { + assert_eq!( + server + .client + .config(proto::ConfigRequest { + session_id: session.clone(), + operation, + ..Default::default() + }) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + } + // A stale observed ID cannot match a session that does not exist. Repeated + // requests must not consume backend session capacity while being rejected. + for _ in 0..3 { + let mut stale = request(&session, 0); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.execute_plan(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + } + let mut stale = config_request( + &session, + proto::config_request::operation::OpType::GetOption(proto::config_request::GetOption { + keys: vec!["missing".into()], + }), + ); + stale.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + assert_eq!( + server.client.config(stale).await.unwrap_err().code(), + Code::FailedPrecondition + ); + // Identifier and envelope validation must precede backend session allocation. + assert_eq!(spy.factory_calls.lock().unwrap().len(), 0); + assert_eq!( + server + .client + .release_execute(release(&session, &Uuid::new_v4().to_string())) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert_eq!( + server + .client + .release_execute(release("not-a-uuid", &session)) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + assert_eq!( + server + .client + .release_execute(release(&session, "not-a-uuid")) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + let unknown = Uuid::new_v4().to_string(); + assert_eq!( + server + .client + .release_execute(release(&unknown, &session)) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + let released = server + .client + .release_session(release_session("not-a-uuid")) + .await + .unwrap() + .into_inner(); + assert_eq!(released.session_id, "not-a-uuid"); + assert!(released.server_side_session_id.is_empty()); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 0); + spy.fail_next.store(true, Ordering::SeqCst); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap_err() + .code(), + Code::Unavailable + ); + server.client.config(get_config(&session)).await.unwrap(); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 2); + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + wait_for_count(&spy.sessions_dropped, 1).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + assert_eq!(spy.factory_calls.lock().unwrap().len(), 2); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_all_cancels_nonreattachable_live_executions() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let request = request(&session, LIVE); + let operation = request.operation_id.clone().unwrap(); + let mut stream = start_live(&mut server, request).await; + let released = server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap() + .into_inner(); + assert_eq!(released.operation_id.as_deref(), Some(operation.as_str())); + assert_cancelled(&mut stream).await; + assert_eq!(spy.executions.load(Ordering::SeqCst), 1); + assert_eq!(spy.streams_dropped.load(Ordering::SeqCst), 1); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_streams_drop_on_release_disconnect_session_release_and_shutdown() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let live = reattachable_request(&session, LIVE); + let operation = live.operation_id.clone().unwrap(); + let mut stream = start_live(&mut server, live.clone()).await; + assert_eq!( + server + .client + .execute_plan(live.clone()) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + let mut duplicate = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(1)), + ..Default::default() + }), + ); + duplicate.operation_id = Some(operation.clone()); + assert_eq!( + server + .client + .execute_plan(duplicate) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + assert_eq!(spy.commands.load(Ordering::SeqCst), 0); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); + let mut release_request = release(&session, &operation); + release_request.client_observed_server_side_session_id = Some( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id, + ); + let released = tokio::time::timeout( + Duration::from_secs(3), + server.client.release_execute(release_request.clone()), + ) + .await + .expect("ReleaseAll must finish without consuming the result") + .unwrap() + .into_inner(); + assert_eq!(released.operation_id.as_deref(), Some(operation.as_str())); + assert_eq!( + spy.streams_dropped.load(Ordering::SeqCst), + 1, + "release must wait for backend stream teardown" + ); + let absent = server + .client + .release_execute(release_request) + .await + .unwrap() + .into_inner(); + assert_eq!( + absent.operation_id, None, + "teardown must remove the operation" + ); + assert_cancelled(&mut stream).await; + + let mut abandoned = server + .client + .execute_plan(live.clone()) + .await + .unwrap() + .into_inner(); + assert!(abandoned.message().await.unwrap().unwrap().schema.is_some()); + // Keep the result batch unread while proving the backend stream exists. + wait_for_count(&spy.executions, 2).await; + drop(abandoned); + wait_for_count(&spy.streams_dropped, 2).await; + let mut restarted = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match server.client.execute_plan(live.clone()).await { + Ok(response) => break response.into_inner(), + Err(status) if status.code() == Code::AlreadyExists => { + tokio::time::sleep(Duration::from_millis(10)).await + } + Err(status) => panic!("unexpected restart error: {status}"), + } + } + }) + .await + .expect("disconnected operation must be removed"); + assert!(restarted.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + restarted.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut restarted).await; + wait_for_count(&spy.streams_dropped, 3).await; + + let mut live = start_live(&mut server, reattachable_request(&session, LIVE)).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + assert_cancelled(&mut live).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!( + server + .client + .execute_plan(request(&session, 99)) + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + let other = Uuid::new_v4().to_string(); + let mut live = start_live(&mut server, reattachable_request(&other, LIVE)).await; + server.shutdown.take().unwrap().send(()).unwrap(); + assert_cancelled(&mut live).await; + server.stop().await; + wait_for_count(&spy.streams_dropped, 5).await; + wait_for_count(&spy.sessions_dropped, 2).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_releases_backend_session_while_a_result_remains_unread() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let (mut client, connection) = h2::client::Builder::new() + .initial_window_size(1024) + .initial_connection_window_size(1024) + .handshake::<_, tonic::codegen::Bytes>( + tokio::net::TcpStream::connect(server.address) + .await + .unwrap(), + ) + .await + .unwrap(); + let connection = tokio::spawn(async move { + let _ = connection.await; + }); + let request = request(&Uuid::new_v4().to_string(), FLOODED_RESULT).encode_to_vec(); + let mut payload = vec![0]; + payload.extend_from_slice(&(request.len() as u32).to_be_bytes()); + payload.extend_from_slice(&request); + let request = tonic::codegen::http::Request::builder() + .method("POST") + .uri(format!( + "http://{}/spark.connect.SparkConnectService/ExecutePlan", + server.address + )) + .header("content-type", "application/grpc") + .body(()) + .unwrap(); + let (response, mut upload) = client.send_request(request, false).unwrap(); + upload.send_data(payload.into(), true).unwrap(); + let response = tokio::time::timeout(Duration::from_secs(5), response) + .await + .expect("response headers must arrive") + .unwrap(); + // Keep the body and HTTP/2 connection alive without reading or releasing + // capacity. Each 512 KiB batch exceeds the stream's 1 KiB receive window. + tokio::time::timeout(Duration::from_secs(5), async { + while spy.batches_polled.load(Ordering::SeqCst) < 4 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the frontend result queue must fill behind the unread peer"); + assert_eq!(spy.sessions_dropped.load(Ordering::SeqCst), 0); + server.shutdown.take().unwrap().send(()).unwrap(); + // Shutdown must release the session registry and the producer's session + // reference before waiting to send a terminal error through the full queue. + wait_for_count(&spy.streams_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert!(!response.body().is_end_stream()); + drop(response); + drop(upload); + drop(client); + connection.abort(); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_reservation_precedes_prepare_and_cleans_up_after_cancellation_and_errors() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let blocked = request(&session, BLOCKED_PREPARE); + let operation = blocked.operation_id.clone().unwrap(); + let mut pending_client = server.client.clone(); + let pending = tokio::spawn(async move { pending_client.execute_plan(blocked).await }); + wait_for_count(&spy.prepares, 1).await; + let mut duplicate = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(1)), + ..Default::default() + }), + ); + duplicate.operation_id = Some(operation.clone()); + assert_eq!( + server + .client + .execute_plan(duplicate) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + assert_eq!(spy.commands.load(Ordering::SeqCst), 0); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_eq!(spy.blocked_prepares_dropped.load(Ordering::SeqCst), 1); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), pending) + .await + .unwrap() + .unwrap() + .unwrap_err() + .code(), + Code::Cancelled + ); + + let mut retry = request(&session, 99); + retry.operation_id = Some(operation); + let mut stream = server + .client + .execute_plan(retry) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![999]); + let rejected = request(&session, -1); + let mut retry = request(&session, 99); + retry.operation_id.clone_from(&rejected.operation_id); + assert_eq!( + server + .client + .execute_plan(rejected) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut stream = server + .client + .execute_plan(retry) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut stream).await), vec![999]); + server.stop().await; +} + +async fn start_blocked_prepare( + server: &Server, + spy: &Spy, + session: &str, +) -> tokio::task::JoinSet { + let blocked = request(session, BLOCKED_PREPARE); + let mut pending_client = server.client.clone(); + // Abort the client task on a failed assertion rather than leaving an RPC running. + let mut pending = tokio::task::JoinSet::new(); + pending.spawn(async move { pending_client.execute_plan(blocked).await.unwrap_err() }); + wait_for_count(&spy.prepares, 1).await; + pending +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn session_release_cancels_pending_prepare_and_drops_backend_session() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut pending = start_blocked_prepare(&server, &spy, &session).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + + let status = tokio::time::timeout(Duration::from_secs(5), pending.join_next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(status.code(), Code::Cancelled); + assert_eq!(status.message(), "session released"); + wait_for_count(&spy.blocked_prepares_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!(spy.executions.load(Ordering::SeqCst), 0); + // Keep server.client connected until cancellation and backend teardown are proven. + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_cancels_pending_prepare_before_transport_drain_expires() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut pending = start_blocked_prepare(&server, &spy, &session).await; + + // Cancellation must finish before the two-second transport drain can drop the RPC. + tokio::time::timeout(Duration::from_secs(1), async { + server.shutdown.take().unwrap().send(()).unwrap(); + let status = pending.join_next().await.unwrap().unwrap(); + assert_eq!(status.code(), Code::Cancelled); + assert_eq!(status.message(), "session released"); + wait_for_count(&spy.blocked_prepares_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!(spy.executions.load(Ordering::SeqCst), 0); + }) + .await + .expect("pending preparation must cancel before the transport drain expires"); + // Keep server.client connected until cancellation and backend teardown are proven. + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_and_shutdown_cancel_backend_metadata_futures_before_dropping_sessions() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut analysis_client = server.client.clone(); + let analysis_session = session.clone(); + let analysis = tokio::spawn(async move { + analysis_client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: analysis_session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsStreaming( + proto::analyze_plan_request::IsStreaming { + plan: request(&analysis_session, 99).plan, + }, + )), + ..Default::default() + }) + .await + }); + wait_for_count(&spy.analyses, 1).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + let status = tokio::time::timeout(Duration::from_secs(5), analysis) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(status.code(), Code::Cancelled); + wait_for_count(&spy.blocked_analyses_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!( + *spy.metadata_drop_observations.lock().unwrap(), + vec![(1, 0)] + ); + + let session = Uuid::new_v4().to_string(); + let mut config_client = server.client.clone(); + let config = tokio::spawn(async move { + config_client + .config(config_request( + &session, + proto::config_request::operation::OpType::Get(proto::config_request::Get { + keys: vec!["fake.block".into()], + }), + )) + .await + }); + wait_for_count(&spy.configs, 1).await; + server.shutdown.take().unwrap().send(()).unwrap(); + let status = tokio::time::timeout(Duration::from_secs(5), config) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(status.code(), Code::Cancelled); + wait_for_count(&spy.blocked_configs_dropped, 1).await; + server.stop().await; + wait_for_count(&spy.sessions_dropped, 2).await; + assert_eq!( + *spy.metadata_drop_observations.lock().unwrap(), + vec![(1, 0), (1, 1)] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn slow_backend_drop_does_not_hold_the_global_session_registry_lock() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + // Channel clones share an HTTP/2 connection. Establish a new TCP connection + // before blocking Drop so the probe isolates the global registry lock. + let mut independent = SparkConnectServiceClient::connect(format!("http://{}", server.address)) + .await + .unwrap(); + let slow_session = Uuid::new_v4().to_string(); + let gate = Arc::new(DropGate::default()); + let unblock = UnblockDrop(gate.clone()); + *spy.slow_drop.lock().unwrap() = Some((slow_session.clone(), gate.clone())); + server + .client + .config(get_config(&slow_session)) + .await + .unwrap(); + let mut release_client = server.client.clone(); + let released = tokio::spawn(async move { + release_client + .release_session(release_session(&slow_session)) + .await + }); + // Store outcomes without asserting while Drop is blocked. Its own ten-second + // deadline and the RAII unblock guard also cover failures in this test. + let entered = tokio::time::timeout(Duration::from_secs(2), async { + while gate.entered.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + let other = tokio::time::timeout( + Duration::from_secs(1), + independent.config(get_config(&Uuid::new_v4().to_string())), + ) + .await; + let still_blocked = !gate.exited.load(Ordering::SeqCst); + drop(unblock); + let released = tokio::time::timeout(Duration::from_secs(5), released).await; + server.stop().await; + + assert!(entered.is_ok(), "slow Drop must start"); + other + .expect("another session must remain responsive while Drop is blocked") + .unwrap(); + assert!( + still_blocked, + "the independent request must complete before Drop exits" + ); + assert!( + !gate.timed_out.load(Ordering::SeqCst), + "Drop must be explicitly unblocked" + ); + released.unwrap().unwrap().unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn frontend_rejects_oversized_analysis_and_batch_type_mismatches() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let status = server + .client + .analyze_plan(proto::AnalyzePlanRequest { + session_id: session.clone(), + user_context: user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::Schema( + proto::analyze_plan_request::Schema { + plan: request(&session, 99).plan, + }, + )), + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!(status.code(), Code::ResourceExhausted); + + let mut mismatch = server + .client + .execute_plan(request(&session, MISMATCHED_BATCH)) + .await + .unwrap() + .into_inner(); + let header = mismatch.message().await.unwrap().unwrap(); + assert!(header.schema.is_some()); + assert!(header.response_type.is_none()); + // The very next event must be the error: no inconsistent Arrow payload or + // successful completion may escape after the declared Int64 schema. + assert_eq!(mismatch.message().await.unwrap_err().code(), Code::Internal); + + let mut compatible = server + .client + .execute_plan(reattachable_request(&session, ANNOTATED_BATCH)) + .await + .unwrap() + .into_inner(); + let responses = collect(&mut compatible).await; + assert_eq!(values(&responses), vec![999]); + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn frontend_rejects_mismatched_and_invalid_ipc_before_forwarding() { + let mut server = start_server(Arc::new(Spy::default())).await; + let session = Uuid::new_v4().to_string(); + for (selector, message) in [ + ( + MISMATCHED_IPC_BATCH, + "engine batch schema differs from its declared schema", + ), + (INVALID_IPC_BATCH, "invalid engine IPC:"), + ] { + let mut stream = server + .client + .execute_plan(request(&session, selector)) + .await + .unwrap() + .into_inner(); + let header = stream.message().await.unwrap().unwrap(); + assert!(header.schema.is_some()); + assert!(header.response_type.is_none()); + // The next event is the schema-validation error, before any IPC payload + // or successful completion can escape the declared schema header. + let status = stream.message().await.unwrap_err(); + assert_eq!(status.code(), Code::Internal); + assert!(status.message().starts_with(message), "{status}"); + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn panicking_execution_reports_internal_after_header_or_partial_result() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + for selector in [PANICKING_EXECUTION, PANICKING_STREAM, PANICKING_STREAM_DROP] { + let mut stream = server + .client + .execute_plan(request(&session, selector)) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + if selector != PANICKING_EXECUTION { + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + } + let error = tokio::time::timeout(Duration::from_secs(5), stream.message()) + .await + .unwrap() + .unwrap_err(); + assert_eq!(error.code(), Code::Internal); + assert!(error.message().contains("panicked")); + } + let mut recovered = server + .client + .execute_plan(request(&session, 1)) + .await + .unwrap() + .into_inner(); + assert_eq!(values(&collect(&mut recovered).await), vec![999]); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn oversized_command_payload_fails_before_sending_any_success_message() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy).await; + let session = Uuid::new_v4().to_string(); + let command = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(proto::Relation { + rel_type: Some(proto::relation::RelType::Read(proto::Read { + read_type: Some(proto::read::ReadType::NamedTable(proto::read::NamedTable { + unparsed_identifier: "x".repeat(3 * 1024 * 1024 + 1), + ..Default::default() + })), + ..Default::default() + })), + ..Default::default() + }), + ..Default::default() + }), + ); + assert_eq!( + server + .client + .execute_plan(command) + .await + .unwrap_err() + .code(), + Code::ResourceExhausted + ); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_all_timeout_keeps_cancellation_and_operation_ownership_until_drain() { + let spy = Arc::new(Spy::default()); + let gate = Arc::new(DropGate::default()); + *spy.stream_drop_gate.lock().unwrap() = Some(gate.clone()); + let unblock = UnblockDrop(gate.clone()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let original = reattachable_request(&session, LIVE); + let operation = original.operation_id.clone().unwrap(); + let mut stream = server + .client + .execute_plan(original.clone()) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert!(matches!( + stream.message().await.unwrap().unwrap().response_type, + Some(ResponseType::ArrowBatch(_)) + )); + let error = server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::DeadlineExceeded); + assert!(!gate.exited.load(Ordering::SeqCst)); + assert_eq!( + server + .client + .execute_plan(original.clone()) + .await + .unwrap_err() + .code(), + Code::AlreadyExists + ); + drop(unblock); + assert_cancelled(&mut stream).await; + assert!(!gate.timed_out.load(Ordering::SeqCst)); + *spy.stream_drop_gate.lock().unwrap() = None; + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + let mut recovered = server + .client + .execute_plan(original) + .await + .unwrap() + .into_inner(); + assert!(recovered.message().await.unwrap().unwrap().schema.is_some()); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut recovered).await; + server.stop().await; +} + +fn identified_request(session: &str, operation: &str) -> proto::ExecutePlanRequest { + let mut request = request(session, 1); + request.operation_id = Some(operation.into()); + request +} + +fn local_analysis(session: &str) -> proto::AnalyzePlanRequest { + proto::AnalyzePlanRequest { + session_id: session.into(), + user_context: requests::user("alice"), + analyze: Some(proto::analyze_plan_request::Analyze::IsLocal( + Default::default(), + )), + ..Default::default() + } +} + +async fn collect_identified(server: &mut Server, session: &str, operation: &str) { + let mut request = reattachable_request(session, 1); + request.operation_id = Some(operation.into()); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let output = collect(&mut stream).await; + assert!(matches!( + output.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); +} + +#[tokio::test] +async fn invalid_ids_do_not_create_or_release_sessions() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let uuid = Uuid::parse_str("abcdef12-3456-4789-abcd-0123456789ab").unwrap(); + let session = uuid.to_string(); + let operation = Uuid::new_v4().to_string(); + let original = server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id; + for invalid in [ + uuid.simple().to_string(), + uuid.braced().to_string(), + uuid.urn().to_string(), + "1-2-3-4".into(), + "1-2-3-4-5-6".into(), + ] { + assert_eq!( + server + .client + .config(get_config(&invalid)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .analyze_plan(local_analysis(&invalid)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .execute_plan(identified_request(&invalid, &operation)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .release_execute(release(&invalid, &operation)) + .await + .unwrap_err() + .code(), + Code::NotFound + ); + let released = server + .client + .release_session(release_session(&invalid)) + .await + .unwrap() + .into_inner(); + assert_eq!(released.session_id, invalid); + assert!(released.server_side_session_id.is_empty()); + assert_eq!(spy.sessions_dropped.load(Ordering::SeqCst), 0); + assert_eq!( + server + .client + .execute_plan(identified_request(&Uuid::new_v4().to_string(), &invalid)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + assert_eq!( + server + .client + .config(get_config(&session)) + .await + .unwrap() + .into_inner() + .server_side_session_id, + original + ); + } + assert_eq!(spy.factory_calls.lock().unwrap().len(), 1); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 0); + collect_identified(&mut server, &session, &operation).await; + server.stop().await; +} + +#[tokio::test] +async fn session_uuid_spellings_keep_distinct_backends_and_response_ids() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let sessions = [ + "abcdef01-0002-0003-0004-000000000005", + "ABCDEF01-0002-0003-0004-000000000005", + "abcdef01-2-3-4-5", + ]; + let mut server_ids = Vec::new(); + for (index, session) in sessions.iter().enumerate() { + let response = server + .client + .config(config_request( + session, + proto::config_request::operation::OpType::Set(proto::config_request::Set { + pairs: vec![proto::KeyValue { + key: "spark.fake.setting".into(), + value: Some(index.to_string()), + }], + silent: Some(false), + }), + )) + .await + .unwrap() + .into_inner(); + assert_eq!(response.session_id, *session); + assert!(!server_ids.contains(&response.server_side_session_id)); + server_ids.push(response.server_side_session_id); + } + assert_eq!( + *spy.factory_calls.lock().unwrap(), + sessions + .iter() + .map(|session| ("alice".into(), session.to_string())) + .collect::>() + ); + for (index, session) in sessions.iter().enumerate() { + let configured = server + .client + .config(get_config(session)) + .await + .unwrap() + .into_inner(); + assert_eq!(configured.session_id, *session); + assert_eq!(configured.server_side_session_id, server_ids[index]); + assert_eq!(configured.pairs[0].value, Some(index.to_string())); + let analysis = server + .client + .analyze_plan(local_analysis(session)) + .await + .unwrap() + .into_inner(); + assert_eq!(analysis.session_id, *session); + assert_eq!(analysis.server_side_session_id, server_ids[index]); + let mut stream = server + .client + .execute_plan(request(session, 1)) + .await + .unwrap() + .into_inner(); + let responses = collect(&mut stream).await; + assert_eq!(values(&responses), vec![999]); + assert!(responses.iter().all(|response| { + response.session_id == *session && response.server_side_session_id == server_ids[index] + })); + } + let released = server + .client + .release_session(release_session(sessions[1])) + .await + .unwrap() + .into_inner(); + assert_eq!(released.session_id, sessions[1]); + assert_eq!(released.server_side_session_id, server_ids[1]); + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!( + server + .client + .config(get_config(sessions[1])) + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + for index in [0, 2] { + let response = server + .client + .config(get_config(sessions[index])) + .await + .unwrap() + .into_inner(); + assert_eq!(response.server_side_session_id, server_ids[index]); + assert_eq!(response.pairs[0].value, Some(index.to_string())); + } + assert_eq!(spy.factory_calls.lock().unwrap().len(), sessions.len()); + server.stop().await; +} + +#[tokio::test] +async fn execute_normalizes_operation_ids_while_release_looks_up_raw_ids() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let operation = "abcdef01-0002-0003-0004-000000000005"; + let aliases = [ + "ABCDEF01-0002-0003-0004-000000000005", + "abcdef01-2-3-4-5", + "+abcdef01-+2-+3-+4-+5", + "1abcdef01-10002-3-4-5", + ]; + let mut live = reattachable_request(&session, LIVE); + live.operation_id = Some(aliases[0].into()); + let mut stream = server.client.execute_plan(live).await.unwrap().into_inner(); + let header = stream.message().await.unwrap().unwrap(); + assert!(header.schema.is_some()); + assert_eq!(header.operation_id, operation); + let server_id = header.server_side_session_id; + let batch = stream.message().await.unwrap().unwrap(); + assert!(matches!( + batch.response_type, + Some(ResponseType::ArrowBatch(_)) + )); + assert_eq!(batch.operation_id, operation); + for alias in aliases { + let mut duplicate = command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(1)), + ..Default::default() + }), + ); + duplicate.operation_id = Some(alias.into()); + assert_eq!( + server + .client + .execute_plan(duplicate) + .await + .unwrap_err() + .code(), + Code::AlreadyExists, + "ExecutePlan must reserve the canonical operation ID for {alias}" + ); + } + assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); + assert_eq!(spy.commands.load(Ordering::SeqCst), 0); + for unmatched in aliases.into_iter().chain(["not-a-uuid"]) { + let mut request = release(&session, unmatched); + request.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + let response = server + .client + .release_execute(request) + .await + .unwrap() + .into_inner(); + assert_eq!(response.session_id, session); + assert_eq!(response.server_side_session_id, server_id); + assert_eq!(response.operation_id, None); + } + assert!( + tokio::time::timeout(Duration::from_millis(50), stream.message()) + .await + .is_err(), + "unmatched raw ReleaseExecute IDs must leave the live execution open" + ); + assert_eq!(spy.streams_dropped.load(Ordering::SeqCst), 0); + for kind in [ + Some(proto::release_execute_request::Release::ReleaseUntil( + proto::release_execute_request::ReleaseUntil { + response_id: batch.response_id.clone(), + }, + )), + None, + ] { + let mut request = release(&session, operation); + request.release = kind; + let response = server + .client + .release_execute(request) + .await + .unwrap() + .into_inner(); + assert_eq!(response.session_id, session); + assert_eq!(response.server_side_session_id, server_id); + assert_eq!(response.operation_id.as_deref(), Some(operation)); + assert!( + tokio::time::timeout(Duration::from_millis(50), stream.message()) + .await + .is_err(), + "ReleaseUntil and a missing release kind must leave the live execution open" + ); + assert_eq!(spy.streams_dropped.load(Ordering::SeqCst), 0); + } + let stale_server_id = Uuid::new_v4().to_string(); + let mut request = release(&session, operation); + request.client_observed_server_side_session_id = Some(stale_server_id.clone()); + let response = server + .client + .release_execute(request) + .await + .unwrap() + .into_inner(); + assert_eq!(response.operation_id.as_deref(), Some(operation)); + assert_eq!(response.server_side_session_id, server_id); + assert_ne!(response.server_side_session_id, stale_server_id); + assert_cancelled(&mut stream).await; + assert_eq!(spy.streams_dropped.load(Ordering::SeqCst), 1); + server.stop().await; +} + +#[tokio::test] +async fn result_complete_requires_any_true_reattach_option_for_queries_and_commands() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + for flags in [ + vec![], + vec![false], + vec![true], + vec![false, true], + vec![true, false], + vec![false, false], + ] { + let complete = flags.iter().any(|&flag| flag); + for operation in 0..3 { + let mut request = match operation { + 0 => request(&session, 1), + 1 => command_request( + &session, + proto::command::CommandType::SqlCommand(proto::SqlCommand { + input: Some(relation(1)), + ..Default::default() + }), + ), + _ => command_request( + &session, + proto::command::CommandType::CreateDataframeView( + proto::CreateDataFrameViewCommand { + input: Some(relation(1)), + name: "completion_view".into(), + is_global: false, + replace: true, + }, + ), + ), + }; + request.request_options = flags + .iter() + .map(|&reattachable| proto::execute_plan_request::RequestOption { + request_option: Some( + proto::execute_plan_request::request_option::RequestOption::ReattachOptions( + proto::ReattachOptions { reattachable }, + ), + ), + }) + .collect(); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let responses = collect(&mut stream).await; + let completions = responses + .iter() + .filter(|response| { + matches!( + response.response_type, + Some(ResponseType::ResultComplete(_)) + ) + }) + .count(); + assert_eq!( + completions, + usize::from(complete), + "options={flags:?}, operation={operation}" + ); + if complete { + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + } + match operation { + 0 => assert_eq!(values(&responses), vec![999]), + 1 => assert!(matches!( + responses[0].response_type, + Some(ResponseType::SqlCommandResult(_)) + )), + _ => assert_eq!(responses.len(), usize::from(complete)), + } + } + } + assert_eq!(spy.executions.load(Ordering::SeqCst), 6); + assert_eq!(spy.commands.load(Ordering::SeqCst), 12); + server.stop().await; +} + +#[tokio::test] +async fn backend_request_panics_have_internal_status_and_leave_the_session_usable() { + for call in [Call::Prepare, Call::Analyze, Call::Config] { + for at in [PanicAt::Construction, PanicAt::Poll] { + let spy = Arc::new(Spy::default()); + *spy.request_panic.lock().unwrap() = Some((call, at)); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let operation = Uuid::new_v4().to_string(); + let status = match call { + Call::Prepare => server + .client + .execute_plan(identified_request(&session, &operation)) + .await + .unwrap_err(), + Call::Analyze => server + .client + .analyze_plan(local_analysis(&session)) + .await + .unwrap_err(), + Call::Config => server + .client + .config(get_config(&session)) + .await + .unwrap_err(), + }; + assert_eq!(status.code(), Code::Internal); + assert_eq!(status.message(), "engine request panicked"); + server.client.config(get_config(&session)).await.unwrap(); + server + .client + .analyze_plan(local_analysis(&session)) + .await + .unwrap(); + // Reuse the operation ID after a preparation panic: its reservation + // must have been released before the failed RPC returned. + collect_identified(&mut server, &session, &operation).await; + assert_eq!(spy.factory_calls.lock().unwrap().len(), 1); + server.stop().await; + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn release_cancels_an_arrow_batch_between_byte_chunks() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut request = reattachable_request(&session, CHUNKED_RESULT); + let operation = request.operation_id.clone().unwrap(); + request.request_options.push(chunking(true, Some(1024))); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + let first = stream.message().await.unwrap().unwrap(); + let Some(ResponseType::ArrowBatch(chunk)) = first.response_type else { + panic!("first Arrow chunk") + }; + assert_eq!(chunk.chunk_index, Some(0)); + assert!(chunk.num_chunks_in_batch.unwrap() > 4000); + assert_eq!(chunk.row_count, 512 * 1024); + assert_eq!(chunk.start_offset, Some(0)); + server + .client + .release_execute(release(&session, &operation)) + .await + .unwrap(); + assert_cancelled(&mut stream).await; + wait_for_count(&spy.streams_dropped, 1).await; + assert_eq!(spy.executions.load(Ordering::SeqCst), 1); + server.stop().await; +} diff --git a/crates/spark-connect-server/tests/protocol.rs b/crates/spark-connect-server/tests/protocol.rs new file mode 100644 index 000000000..b8423758c --- /dev/null +++ b/crates/spark-connect-server/tests/protocol.rs @@ -0,0 +1,942 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Verify observable Spark Connect wire behavior independently of DataFrame conveniences. + +#[path = "support/arrow.rs"] +mod arrow_support; +#[path = "support/requests.rs"] +mod requests; +#[path = "support/server.rs"] +mod server; +mod support; + +use std::collections::HashSet; +use std::io::Cursor; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{ + Array, ArrayRef, BinaryViewArray, Int64Array, LargeBinaryArray, LargeListArray, + LargeStringArray, ListArray, MapArray, StringViewArray, StructArray, +}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::ipc::reader::StreamReader; +use arrow::ipc::CompressionType; +use arrow::record_batch::RecordBatch; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use spark_connect_proto as proto; + +use arrow_support::{assembled_batches, chunking, encoded, DEFAULT_ARROW_CHUNK_SIZE}; +use requests::{reattachable_request, request}; +use server::Server; +use support::{Batch, FixtureEngine}; +use tonic::Code; +use uuid::Uuid; + +async fn start_server() -> Server { + Server::start_with_engine(Arc::new(FixtureEngine::default())).await +} + +#[tokio::test] +async fn inbound_requests_use_the_spark_128_mib_limit() { + let mut server = start_server().await; + let session = Uuid::new_v4().to_string(); + const MAX_REQUEST_BYTES: usize = 128 * 1024 * 1024; + // Allocate the next payload only after the previous request has completed. + for encoded_bytes in [MAX_REQUEST_BYTES, MAX_REQUEST_BYTES + 1] { + let mut input = request(&session, 0); + // Client metadata is ignored by execution. Account for its length + // delimiter to test the exact wire boundary without a large result. + input.client_type = Some(String::new()); + let fixed_bytes = input.encoded_len() - prost::length_delimiter_len(0); + let metadata_bytes = + encoded_bytes - fixed_bytes - prost::length_delimiter_len(encoded_bytes); + input.client_type = Some("x".repeat(metadata_bytes)); + assert_eq!(input.encoded_len(), encoded_bytes); + let response = server.client.execute_plan(input).await; + if encoded_bytes == MAX_REQUEST_BYTES { + let mut stream = response.unwrap().into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + while stream.message().await.unwrap().is_some() {} + } else { + assert_eq!(response.unwrap_err().code(), Code::OutOfRange); + } + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn arrow_stream_preserves_schema_ids_offsets_and_empty_results() { + let mut server = start_server().await; + let session = Uuid::new_v4().to_string(); + let mut server_session = None; + for end in [0, 25_000] { + let request = reattachable_request(&session, end); + let operation = request.operation_id.clone().unwrap(); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let mut response_ids = HashSet::new(); + let (mut rows, mut batches, mut schemas, mut complete) = (0, 0, 0, false); + while let Some(response) = stream.message().await.unwrap() { + assert!(!complete, "ResultComplete must be the last response"); + assert_eq!(response.session_id, session); + assert_eq!(response.operation_id, operation); + Uuid::parse_str(&response.response_id).unwrap(); + assert!(response_ids.insert(response.response_id)); + Uuid::parse_str(&response.server_side_session_id).unwrap(); + let stable = server_session.get_or_insert(response.server_side_session_id.clone()); + assert_eq!(*stable, response.server_side_session_id); + if let Some(schema) = response.schema { + let Some(proto::data_type::Kind::Struct(schema)) = schema.kind else { + panic!("struct schema"); + }; + assert_eq!(schema.fields.len(), 1); + assert_eq!(schema.fields[0].name, "id"); + assert!(!schema.fields[0].nullable); + assert!(matches!( + schema.fields[0].data_type.as_ref().unwrap().kind, + Some(proto::data_type::Kind::Long(_)) + )); + schemas += 1; + } + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(rows)); + let reader = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let mut batch_rows = 0; + for record in reader { + let record = record.unwrap(); + let values = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..values.len() { + assert_eq!(values.value(index), rows + batch_rows + index as i64); + } + batch_rows += record.num_rows() as i64; + } + assert_eq!(batch_rows, batch.row_count); + rows += batch_rows; + batches += 1; + } + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, end); + assert_eq!(schemas, 1); + assert!(complete); + assert!(batches >= 1, "empty execution must still return Arrow data"); + if end > 0 { + // The fixture supplies multiple physical batches; all must reach the wire. + assert!(batches > 1, "large result should stream multiple batches"); + } + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn observed_session_mismatch_and_released_sessions_are_rejected() { + let mut server = start_server().await; + let session = Uuid::new_v4().to_string(); + let mut first = server + .client + .execute_plan(request(&session, 0)) + .await + .unwrap() + .into_inner(); + let original = first + .message() + .await + .unwrap() + .unwrap() + .server_side_session_id; + while first.message().await.unwrap().is_some() {} + let mut mismatch = request(&session, 1); + mismatch.client_observed_server_side_session_id = Some(Uuid::new_v4().to_string()); + let error = server.client.execute_plan(mismatch).await.unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + let release = proto::ReleaseSessionRequest { + session_id: session.clone(), + user_context: requests::user("alice"), + ..Default::default() + }; + let released = server + .client + .release_session(release.clone()) + .await + .unwrap() + .into_inner(); + assert_eq!(released.server_side_session_id, original); + server.client.release_session(release).await.unwrap(); + let error = server + .client + .execute_plan(request(&session, 1)) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + // A separate session on the same transport remains usable. + let mut other = server + .client + .execute_plan(request(&Uuid::new_v4().to_string(), 0)) + .await + .unwrap() + .into_inner(); + while other.message().await.unwrap().is_some() {} + server.stop().await; +} + +async fn collect_result( + server: &mut Server, + request: proto::ExecutePlanRequest, +) -> Vec { + tokio::time::timeout(Duration::from_secs(30), async { + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let mut responses = Vec::new(); + while let Some(response) = stream.message().await.unwrap() { + responses.push(response); + } + assert!(matches!( + responses.last().unwrap().response_type, + Some(ResponseType::ResultComplete(_)) + )); + responses + }) + .await + .expect("fixture execution must terminate") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn negotiated_chunks_preserve_plain_and_compressed_ipc_bytes_and_offsets() { + // Incompressible values keep both compression codecs above the chunk limit. + let mut random = 1u64; + let values = Int64Array::from_iter_values((0..450_000).map(|_| { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + random as i64 + })); + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(values)]).unwrap(); + for compression in [ + None, + Some(CompressionType::ZSTD), + Some(CompressionType::LZ4_FRAME), + ] { + let data = encoded(&batch, compression); + assert!(data.len() > 3 * 1024 * 1024); + let tail = batch.slice(0, 3); + let tail_data = encoded(&tail, compression); + let engine = FixtureEngine::default().with_results( + 42, + schema.clone(), + vec![ + Batch::Ipc(data.clone(), batch.num_rows() as i64), + Batch::Ipc(tail_data.clone(), 3), + ], + ); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + server.client = server + .client + .clone() + .max_decoding_message_size(8 * 1024 * 1024); + for option in [ + None, + Some(chunking(false, Some(1024))), + Some(chunking(true, None)), + Some(chunking(true, Some(3 * 1024 * 1024))), + ] { + let negotiated = option.as_ref().is_some_and(|option| matches!( + &option.request_option, + Some(proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions(options)) + if options.allow_arrow_batch_chunking + )); + let mut request = reattachable_request(&Uuid::new_v4().to_string(), 42); + let limited = option.as_ref().is_some_and(|option| matches!( + &option.request_option, + Some(proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions(options)) + if options.allow_arrow_batch_chunking && options.preferred_arrow_chunk_size.is_some() + )); + request.request_options.extend(option); + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, negotiated); + assert_eq!(output.len(), 2); + assert_eq!(output[0].data.as_ref(), data.as_slice()); + assert_eq!(output[1].data.as_ref(), tail_data.as_slice()); + assert_eq!(output[0].row_count, 450_000); + assert_eq!(output[1].start_offset, Some(450_000)); + for (result, expected) in output.iter().zip([&batch, &tail]) { + let mut reader = StreamReader::try_new(result.data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), *expected); + assert!(reader.next().is_none()); + } + let chunks = responses + .iter() + .filter(|response| { + matches!(response.response_type, Some(ResponseType::ArrowBatch(_))) + }) + .count(); + assert_eq!(chunks, if limited { 3 } else { 2 }); + } + server.stop().await; + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn preferred_chunk_size_uses_the_protocol_range_and_splits_inside_a_large_row() { + use arrow::array::StringArray; + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Utf8, + false, + )])), + vec![Arc::new(StringArray::from(vec![ + "x".repeat(4 * 1024 * 1024) + ]))], + ) + .unwrap(); + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch.clone()]); + let mut options = spark_connect_server::ServerOptions::default(); + options.max_arrow_chunk_size = 3 * 1024 * 1024; + let mut server = Server::start_with_options(Arc::new(engine), options).await; + for preferred in [ + Some(1024), + Some(65_537), + Some(3 * 1024 * 1024), + None, + Some(0), + Some(-1), + Some(1023), + Some(3 * 1024 * 1024 + 1), + Some(i64::MAX), + ] { + let mut request = reattachable_request(&Uuid::new_v4().to_string(), 42); + request.request_options.push(chunking(true, preferred)); + let responses = collect_result(&mut server, request).await; + let payloads: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => Some(batch.data.len()), + _ => None, + }) + .collect(); + assert!(payloads.len() >= 2); + if let Some(size) = preferred.filter(|size| (1024..=3 * 1024 * 1024).contains(size)) { + assert!(payloads.iter().all(|len| *len <= size as usize)); + assert_eq!(payloads[0], size as usize); + } else { + assert_eq!( + payloads.len(), + 2, + "invalid preferences use the server maximum" + ); + } + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert_eq!(output[0].row_count, 1); + let mut reader = StreamReader::try_new(output[0].data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), batch); + assert!(reader.next().is_none()); + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn backend_status_codes_survive_prepare_and_stream_failures() { + let engine = FixtureEngine::default() + .reject(10, Code::Unimplemented) + .reject(11, Code::InvalidArgument) + .stream_error(12, Code::ResourceExhausted); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let session = Uuid::new_v4().to_string(); + for (selector, expected) in [(10, Code::Unimplemented), (11, Code::InvalidArgument)] { + let status = server + .client + .execute_plan(reattachable_request(&session, selector)) + .await + .unwrap_err(); + assert_eq!(status.code(), expected); + } + let mut stream = server + .client + .execute_plan(reattachable_request(&session, 12)) + .await + .unwrap() + .into_inner(); + assert!(stream.message().await.unwrap().unwrap().schema.is_some()); + assert_eq!( + stream.message().await.unwrap_err().code(), + Code::ResourceExhausted + ); + // Backend failures must not poison the protocol session or its transport. + let mut recovered = server + .client + .execute_plan(reattachable_request(&session, 1)) + .await + .unwrap() + .into_inner(); + let (mut rows, mut complete) = (0, false); + while let Some(response) = recovered.message().await.unwrap() { + match response.response_type { + Some(ResponseType::ArrowBatch(batch)) => rows += batch.row_count, + Some(ResponseType::ResultComplete(_)) => complete = true, + None => {} + other => panic!("unexpected result: {other:?}"), + } + } + assert_eq!(rows, 1); + assert!(complete); + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn deeply_nested_requests_follow_the_configured_protobuf_decoder_limit() { + let mut server = start_server().await; + let session = Uuid::new_v4().to_string(); + let mut request = request(&session, 1); + let Some(proto::plan::OpType::Root(mut relation)) = request.plan.take().unwrap().op_type else { + unreachable!() + }; + for _ in 0..60 { + relation = proto::Relation { + rel_type: Some(proto::relation::RelType::Filter(Box::new(proto::Filter { + input: Some(Box::new(relation)), + condition: Some(proto::Expression { + expr_type: Some(proto::expression::ExprType::Literal( + proto::expression::Literal { + literal_type: Some(proto::expression::literal::LiteralType::Boolean( + true, + )), + ..Default::default() + }, + )), + ..Default::default() + }), + }))), + ..Default::default() + }; + } + request.plan = Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation)), + }); + let decoded = proto::ExecutePlanRequest::decode(request.encode_to_vec().as_slice()); + let status = server.client.execute_plan(request).await.unwrap_err(); + match decoded { + Err(error) => { + // The default decoder rejects this before the backend can inspect it. + assert!(error.to_string().contains("recursion"), "{error}"); + assert_eq!(status.code(), Code::Internal); + assert!(status.message().contains("recursion"), "{status}"); + } + Ok(_) => { + // Other workspace crates enable prost/no-recursion-limit. In that + // configuration the unsupported Filter reaches the fixture backend. + assert_eq!(status.code(), Code::Unimplemented); + assert_eq!(status.message(), "fixture requires a range selector"); + } + } + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_arrays_are_materialized_before_byte_chunking_including_struct_children() { + use arrow::array::{BinaryArray, BinaryViewArray, StringArray, StringViewArray, StructArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatch; + + let session = Uuid::new_v4().to_string(); + let text = |id: i64| format!("{id:04}{}", "x".repeat(1020)); + let nested_text = |id: i64| format!("n{id:04}{}", "y".repeat(1019)); + let binary = |id: i64| { + let mut bytes = id.to_le_bytes().to_vec(); + bytes.extend(std::iter::repeat_n((id % 251) as u8, 1016)); + bytes + }; + let strings = StringViewArray::from_iter_values((0..4096).map(text)); + let binaries = BinaryViewArray::from_iter_values((0..4096).map(binary)); + let nested_strings = StringViewArray::from_iter_values((0..4096).map(nested_text)); + // These view columns exceed the chunk preference, so their encoded batch + // spans several byte chunks. + for size in [ + strings + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + binaries + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + nested_strings + .data_buffers() + .iter() + .map(|buffer| buffer.len()) + .sum::(), + ] { + assert!(size > 3 * 1024 * 1024); + } + let nested_fields: arrow::datatypes::Fields = + vec![Field::new("text", DataType::Utf8View, false)].into(); + let nested = StructArray::new(nested_fields.clone(), vec![Arc::new(nested_strings)], None); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("text", DataType::Utf8View, false), + Field::new("binary", DataType::BinaryView, false), + Field::new("nested", DataType::Struct(nested_fields), false), + ])), + vec![ + Arc::new(Int64Array::from_iter_values(0..4096)), + Arc::new(strings), + Arc::new(binaries), + Arc::new(nested), + ], + ) + .unwrap(); + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch]); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let mut request = reattachable_request(&session, 42); + request + .request_options + .push(chunking(true, Some(3 * 1024 * 1024))); + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert!(output[0].num_chunks_in_batch.unwrap() > 1); + let batch = &output[0]; + assert_eq!(batch.start_offset, Some(0)); + let records = StreamReader::try_new(Cursor::new(&batch.data), None).unwrap(); + let mut rows = 0; + for record in records { + let record = record.unwrap(); + let ids = record + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(record.column(1).data_type(), &DataType::LargeUtf8); + assert_eq!(record.column(2).data_type(), &DataType::LargeBinary); + let strings = arrow::compute::cast(record.column(1), &DataType::Utf8).unwrap(); + let strings = strings.as_any().downcast_ref::().unwrap(); + let binaries = arrow::compute::cast(record.column(2), &DataType::Binary).unwrap(); + let binaries = binaries.as_any().downcast_ref::().unwrap(); + let nested = record + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(nested.column(0).data_type(), &DataType::LargeUtf8); + let nested_strings = arrow::compute::cast(nested.column(0), &DataType::Utf8).unwrap(); + let nested_strings = nested_strings + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..record.num_rows() { + let id = rows + index as i64; + assert_eq!(ids.value(index), id); + assert_eq!(strings.value(index), text(id)); + assert_eq!(binaries.value(index), binary(id)); + assert_eq!(nested_strings.value(index), nested_text(id)); + } + rows += record.num_rows() as i64; + } + assert_eq!(rows, batch.row_count); + assert_eq!(rows, 4096); + server.stop().await; +} + +fn integers(rows: usize) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + vec![Arc::new(Int64Array::from_iter_values(0..rows as i64))], + ) + .unwrap() +} + +#[tokio::test] +async fn repeated_chunk_options_enable_if_any_and_use_the_first_preference() { + let batch = integers(3000); + let engine = FixtureEngine::default().with_batches(42, batch.schema(), vec![batch.clone()]); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + for (options, size) in [ + ( + vec![chunking(false, Some(1024)), chunking(true, None)], + 1024, + ), + ( + vec![chunking(false, None), chunking(true, Some(2048))], + 2048, + ), + ( + vec![chunking(true, Some(1023)), chunking(true, Some(1024))], + DEFAULT_ARROW_CHUNK_SIZE, + ), + ] { + let mut request = reattachable_request(&Uuid::new_v4().to_string(), 42); + request.request_options.extend(options); + let responses = collect_result(&mut server, request).await; + let payloads: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => Some(batch.data.len()), + _ => None, + }) + .collect(); + if size < DEFAULT_ARROW_CHUNK_SIZE { + assert_eq!(payloads[0], size); + } else { + assert_eq!(payloads.len(), 1); + } + assert!(payloads.iter().all(|len| *len <= size)); + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + let mut reader = StreamReader::try_new(output[0].data.as_ref(), None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap(), batch); + } + server.stop().await; +} + +async fn collect_small_slices(backing: RecordBatch) -> Vec { + let batches = (0..20) + .map(|index| backing.slice(index * 1000, 1000)) + .collect(); + let engine = FixtureEngine::default().with_batches(42, backing.schema(), batches); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let output = collect_result( + &mut server, + reattachable_request(&Uuid::new_v4().to_string(), 42), + ) + .await; + let (mut batches, mut rows) = (Vec::new(), 0); + for message in output { + // A small slice must not serialize the much larger retained backing allocation. + assert!(message.encoded_len() <= 3 * 1024 * 1024); + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + message.response_type + { + assert_eq!(batch.start_offset, Some(rows)); + assert_eq!(batch.row_count, 1000); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + batches.push(decoded); + assert!(reader.next().is_none()); + rows += batch.row_count; + } + } + assert_eq!(batches.len(), 20); + assert_eq!(rows, 20_000); + server.stop().await; + batches +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn slices_of_a_large_allocation_remain_one_response_per_small_batch() { + for (index, batch) in collect_small_slices(integers(10_000_000)) + .await + .iter() + .enumerate() + { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let start = index as i64 * 1000; + assert!(values.values().iter().copied().eq(start..start + 1000)); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn nested_slices_preserve_referenced_child_values() { + // The parent needs only 20,000 rows to retain the full 80 MB child. Exercise + // each offset width and maps sequentially, sharing the same allocation. + let child: ArrayRef = Arc::new(Int64Array::from_iter_values(0..10_000_000)); + for kind in 0..3 { + let item = Arc::new(Field::new("item", DataType::Int64, false)); + let column: ArrayRef = match kind { + 0 => Arc::new(ListArray::new( + item, + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + child.clone(), + None, + )), + 1 => Arc::new(LargeListArray::new( + item, + OffsetBuffer::new((0..=20_000i64).collect::>().into()), + child.clone(), + None, + )), + _ => { + let fields = vec![ + Field::new("key", DataType::Int64, false), + Field::new("value", DataType::Int64, false), + ]; + let entries = StructArray::new( + fields.clone().into(), + vec![child.clone(), child.clone()], + None, + ); + Arc::new(MapArray::new( + Arc::new(Field::new( + "entries", + DataType::Struct(fields.into()), + false, + )), + OffsetBuffer::new((0..=20_000i32).collect::>().into()), + entries, + None, + false, + )) + } + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "nested", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0i64; + for batch in batches { + for index in 0..batch.num_rows() { + let value = match kind { + 0 => batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + 1 => batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index), + _ => { + let entry = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(index); + assert_eq!(entry.len(), 1); + let key = entry + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(key.value(0), row); + entry.column(1).clone() + } + }; + let value = value.as_any().downcast_ref::().unwrap(); + assert_eq!(value.len(), 1); + assert_eq!(value.value(0), row); + row += 1; + } + } + assert_eq!(row, 20_000); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_slices_preserve_current_values() { + fn text(row: usize) -> String { + format!("{row:010}{}", "y".repeat(30)) + } + + let values = StringViewArray::from_iter_values((0..1_000_000).map(text)); + let (views, buffers, nulls) = values.into_parts(); + for binary in [false, true] { + let column: ArrayRef = if binary { + Arc::new(BinaryViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + } else { + Arc::new(StringViewArray::new( + views.clone(), + buffers.clone(), + nulls.clone(), + )) + }; + let schema = Arc::new(Schema::new(vec![Field::new( + "view", + column.data_type().clone(), + false, + )])); + let backing = RecordBatch::try_new(schema, vec![column]).unwrap(); + let batches = collect_small_slices(backing).await; + let mut row = 0; + for batch in batches { + for index in 0..batch.num_rows() { + let expected = text(row); + if binary { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.value(index), expected.as_bytes()); + } else { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.value(index), expected); + } + row += 1; + } + } + assert_eq!(row, 20_000); + } +} + +#[cfg(target_pointer_width = "64")] +#[tokio::test] +async fn unrepresentable_engine_batches_preserve_preceding_results_and_fail_cleanly() { + use arrow::array::NullArray; + use arrow::record_batch::RecordBatchOptions; + + let rows = i64::MAX as usize + 1; + for null_column in [false, true] { + let oversized = if null_column { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("nulls", DataType::Null, true)])), + vec![Arc::new(NullArray::new(rows))], + ) + .unwrap() + } else { + RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + vec![], + &RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap() + }; + let engine = FixtureEngine::default().with_batches( + 42, + oversized.schema(), + vec![oversized.slice(0, 3), oversized], + ); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let mut stream = server + .client + .execute_plan(reattachable_request(&Uuid::new_v4().to_string(), 42)) + .await + .unwrap() + .into_inner(); + let (status, counts) = tokio::time::timeout(Duration::from_secs(15), async { + let mut counts = Vec::new(); + loop { + match stream.message().await { + Ok(Some(message)) => match message.response_type { + Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) => { + assert_eq!(batch.start_offset, Some(0)); + let mut reader = StreamReader::try_new(&batch.data[..], None).unwrap(); + assert_eq!(reader.next().unwrap().unwrap().num_rows(), 3); + assert!(reader.next().is_none()); + counts.push(batch.row_count); + } + Some(proto::execute_plan_response::ResponseType::ResultComplete(_)) => { + panic!("unrepresentable output must not complete successfully"); + } + _ => {} + }, + Err(status) => break (status, counts), + Ok(None) => panic!("expected row-count overflow status"), + } + } + }) + .await + .unwrap(); + assert_eq!(status.code(), Code::ResourceExhausted); + assert_eq!(status.message(), "result batch row count exceeds i64"); + assert_eq!(counts, vec![3]); + server.stop().await; + } +} + +#[tokio::test] +async fn configured_maximum_bounds_payloads_and_zero_disables_chunking() { + assert_eq!( + spark_connect_server::ServerOptions::default().max_arrow_chunk_size, + DEFAULT_ARROW_CHUNK_SIZE + ); + let batch = integers(5000); + let bytes = encoded(&batch, None); + for maximum in [4096, 0] { + let engine = FixtureEngine::default().with_results( + 42, + batch.schema(), + vec![Batch::Ipc(bytes.clone(), 5000)], + ); + let mut options = spark_connect_server::ServerOptions::default(); + options.max_arrow_chunk_size = maximum; + let mut server = Server::start_with_options(Arc::new(engine), options).await; + for preferred in [None, Some(2048), Some(4096), Some(4097)] { + let mut request = reattachable_request(&Uuid::new_v4().to_string(), 42); + request.request_options.push(chunking(true, preferred)); + let responses = collect_result(&mut server, request).await; + let output = assembled_batches(&responses, maximum != 0); + assert_eq!(output.len(), 1); + assert_eq!(output[0].data.as_ref(), bytes.as_slice()); + let chunks: Vec<_> = responses + .iter() + .filter_map(|response| match &response.response_type { + Some(ResponseType::ArrowBatch(batch)) => { + Some((response.encoded_len(), batch.data.len())) + } + _ => None, + }) + .collect(); + if maximum == 0 { + assert_eq!(chunks.len(), 1); + } else { + let limit = preferred + .filter(|size| (1024..=maximum as i64).contains(size)) + .unwrap_or(maximum as i64) as usize; + assert!(chunks.iter().all(|(_, payload)| *payload <= limit)); + assert_eq!(chunks[0].1, limit); + assert!( + chunks[0].0 > limit, + "the chunk limit excludes the protobuf envelope" + ); + } + } + server.stop().await; + } +} diff --git a/crates/spark-connect-server/tests/schema_consistency.rs b/crates/spark-connect-server/tests/schema_consistency.rs new file mode 100644 index 000000000..023c185fb --- /dev/null +++ b/crates/spark-connect-server/tests/schema_consistency.rs @@ -0,0 +1,481 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Result schemas remain consumable as one Arrow table across output batches. + +#[path = "support/arrow.rs"] +mod arrow_support; +#[path = "support/requests.rs"] +mod requests; +#[path = "support/server.rs"] +mod server; +mod support; + +use std::collections::HashMap; +use std::io::Cursor; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{ + Array, ArrayRef, BinaryViewArray, LargeBinaryArray, LargeListArray, LargeStringArray, + ListArray, MapArray, StringArray, StringViewArray, StructArray, +}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; +use arrow::record_batch::RecordBatch; +use prost::Message; +use spark_connect_proto as proto; +use tonic::{Code, Status}; +use uuid::Uuid; + +use arrow_support::{assembled_batches, chunking}; +use server::Server; +use support::{Batch, FixtureEngine}; + +const CHUNK_PREFERENCE: usize = 3 * 1024 * 1024; +const LARGE_ROWS: usize = 4096; + +struct BatchCase { + schema: SchemaRef, + batches: Vec, +} + +fn ipc(batch: &RecordBatch) -> Vec { + let mut data = Vec::new(); + let mut writer = StreamWriter::try_new(&mut data, batch.schema().as_ref()).unwrap(); + writer.write(batch).unwrap(); + writer.finish().unwrap(); + drop(writer); + data +} + +async fn execute(engine: BatchCase) -> (Vec, Option) { + let engine = FixtureEngine::default().with_results(1, engine.schema, engine.batches); + let mut server = Server::start_with_engine(Arc::new(engine)).await; + let mut request = requests::reattachable_request(&Uuid::new_v4().to_string(), 1); + request + .request_options + .push(chunking(true, Some(CHUNK_PREFERENCE as i64))); + let mut stream = server + .client + .execute_plan(request) + .await + .unwrap() + .into_inner(); + let result = tokio::time::timeout(Duration::from_secs(30), async { + let mut responses = Vec::new(); + loop { + match stream.message().await { + Ok(Some(response)) => { + if let Some(proto::execute_plan_response::ResponseType::ArrowBatch(batch)) = + &response.response_type + { + assert!(batch.data.len() <= CHUNK_PREFERENCE); + assert!(response.encoded_len() <= batch.data.len() + 512); + } else { + // Schema and command envelopes retain their independent 3 MiB cap. + assert!(response.encoded_len() <= 3 * 1024 * 1024); + } + responses.push(response); + } + Ok(None) => break (responses, None), + Err(error) => break (responses, Some(error)), + } + } + }) + .await + .expect("schema fixture execution must terminate"); + server.stop().await; + result +} + +fn output_batches(responses: &[proto::ExecutePlanResponse]) -> Vec { + assembled_batches(responses, true) + .into_iter() + .map(|batch| { + let mut reader = StreamReader::try_new(Cursor::new(batch.data), None).unwrap(); + let decoded = reader.next().unwrap().unwrap(); + assert_eq!(decoded.num_rows() as i64, batch.row_count); + assert!(reader.next().is_none()); + decoded + }) + .collect() +} + +fn assert_schema_error(responses: &[proto::ExecutePlanResponse], status: Option) { + let status = status.expect("a changed output schema must fail"); + assert_eq!(status.code(), Code::Internal); + assert_eq!( + status.message(), + "Arrow output schema changed within one result" + ); + assert!(!completed(responses)); +} + +fn completed(responses: &[proto::ExecutePlanResponse]) -> bool { + responses.iter().any(|response| { + matches!( + response.response_type, + Some(proto::execute_plan_response::ResponseType::ResultComplete( + _ + )) + ) + }) +} + +fn strings(data_type: DataType, nullable: bool) -> RecordBatch { + let values: ArrayRef = match data_type { + DataType::Utf8 => Arc::new(StringArray::from(vec!["value"])), + DataType::LargeUtf8 => Arc::new(LargeStringArray::from(vec!["value"])), + _ => unreachable!(), + }; + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("value", data_type, nullable)])), + vec![values], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn changed_physical_types_and_nullability_fail_before_the_second_batch() { + let first = strings(DataType::Utf8, false); + for second in [ + strings(DataType::LargeUtf8, false), + strings(DataType::Utf8, true), + ] { + let (responses, status) = execute(BatchCase { + schema: first.schema(), + batches: vec![Batch::Arrow(first.clone()), Batch::Arrow(second)], + }) + .await; + assert_schema_error(&responses, status); + let batches = output_batches(&responses); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].schema(), first.schema()); + } +} + +fn metadata_batch(note: &str) -> RecordBatch { + let metadata = HashMap::from([("producer.note".to_string(), note.to_string())]); + let item = + Arc::new(Field::new("item", DataType::LargeUtf8, true).with_metadata(metadata.clone())); + let values = ListArray::new( + item.clone(), + OffsetBuffer::new(vec![0, 1].into()), + Arc::new(LargeStringArray::from(vec!["value"])), + None, + ); + RecordBatch::try_new( + Arc::new(Schema::new_with_metadata( + vec![Field::new("value", DataType::List(item), true).with_metadata(metadata.clone())], + metadata, + )), + vec![Arc::new(values)], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn metadata_can_change_and_batch_schema_can_differ_from_the_declaration() { + let declared = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, false))), + false, + )])); + let (responses, status) = execute(BatchCase { + schema: declared, + batches: vec![ + Batch::Arrow(metadata_batch("first")), + Batch::Arrow(metadata_batch("second")), + ], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let batches = output_batches(&responses); + assert_eq!(batches.len(), 2); + for (batch, note) in batches.iter().zip(["first", "second"]) { + let schema = batch.schema(); + assert_eq!(schema.metadata()["producer.note"], note); + assert_eq!(schema.field(0).metadata()["producer.note"], note); + let DataType::List(item) = schema.field(0).data_type() else { + panic!("expected list output"); + }; + assert_eq!(item.metadata()["producer.note"], note); + assert_eq!(item.data_type(), &DataType::LargeUtf8); + assert!(schema.field(0).is_nullable()); + assert!(item.is_nullable()); + } +} + +fn text(row: usize) -> String { + format!("{row:04}{}", "x".repeat(1020)) +} + +fn views(rows: usize) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Utf8View, + false, + )])), + vec![Arc::new(StringViewArray::from_iter_values( + (0..rows).map(text), + ))], + ) + .unwrap() +} + +fn materialized_view() -> RecordBatch { + RecordBatch::try_new( + strings(DataType::LargeUtf8, false).schema(), + vec![Arc::new(LargeStringArray::from(vec![text(0)]))], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn materialized_views_can_mix_with_matching_arrow_and_ipc_batches() { + let view = views(1); + let large = strings(DataType::LargeUtf8, false); + let materialized = materialized_view(); + let data = ipc(&large); + for use_ipc in [false, true] { + for view_first in [false, true] { + let other = if use_ipc { + Batch::Ipc(data.clone(), 1) + } else { + Batch::Arrow(large.clone()) + }; + let mut batches = vec![Batch::Arrow(view.clone()), other]; + let mut expected = vec![materialized.clone(), large.clone()]; + if !view_first { + batches.reverse(); + expected.reverse(); + } + let (responses, status) = execute(BatchCase { + schema: view.schema(), + batches, + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(responses.last().is_some_and(|response| matches!( + response.response_type, + Some(proto::execute_plan_response::ResponseType::ResultComplete( + _ + )) + ))); + assert_eq!(output_batches(&responses), expected); + if use_ipc { + let output = assembled_batches(&responses, true); + let ipc_index = if view_first { 1 } else { 0 }; + assert_eq!(output[ipc_index].data.as_ref(), data.as_slice()); + } + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn materialized_views_cannot_mix_with_preserved_view_ipc() { + let view = views(1); + let materialized = materialized_view(); + let data = ipc(&view); + for view_first in [false, true] { + let mut batches = vec![Batch::Arrow(view.clone()), Batch::Ipc(data.clone(), 1)]; + if !view_first { + batches.reverse(); + } + let (responses, status) = execute(BatchCase { + schema: view.schema(), + batches, + }) + .await; + assert_schema_error(&responses, status); + let expected = if view_first { + materialized.clone() + } else { + view.clone() + }; + assert_eq!(output_batches(&responses), vec![expected]); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn view_ipc_remains_unchanged_for_small_and_oversized_batches() { + for rows in [2, LARGE_ROWS] { + let batch = views(rows); + let data = ipc(&batch); + assert_eq!(data.len() > CHUNK_PREFERENCE, rows == LARGE_ROWS); + let (responses, status) = execute(BatchCase { + schema: batch.schema(), + batches: vec![Batch::Ipc(data.clone(), rows as i64)], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let output = assembled_batches(&responses, true); + assert_eq!(output.len(), 1); + assert_eq!(output[0].data.as_ref(), data.as_slice()); + let batches = output_batches(&responses); + assert_eq!(batches, vec![batch]); + assert_eq!( + batches[0].schema().field(0).data_type(), + &DataType::Utf8View + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mixing_small_and_oversized_view_ipc_preserves_one_output_schema() { + let small = views(2); + let large = views(LARGE_ROWS); + for large_first in [false, true] { + let expected = if large_first { + vec![large.clone(), small.clone()] + } else { + vec![small.clone(), large.clone()] + }; + let (responses, status) = execute(BatchCase { + schema: small.schema(), + batches: expected + .iter() + .map(|batch| Batch::Ipc(ipc(batch), batch.num_rows() as i64)) + .collect(), + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + assert_eq!(output_batches(&responses), expected); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn arrow_view_children_are_materialized_before_byte_chunking() { + let values: ArrayRef = Arc::new(StringViewArray::from_iter_values((0..LARGE_ROWS).map(text))); + let offsets = OffsetBuffer::new((0..=LARGE_ROWS as i32).collect::>().into()); + let item = Arc::new(Field::new("item", DataType::Utf8View, false)); + let lists = ListArray::new(item.clone(), offsets.clone(), values.clone(), None); + let large_lists = LargeListArray::new( + item, + OffsetBuffer::new((0..=LARGE_ROWS as i64).collect::>().into()), + values.clone(), + None, + ); + let entries = StructArray::new( + vec![ + Field::new("key", DataType::Utf8View, false), + Field::new("value", DataType::BinaryView, false), + ] + .into(), + vec![ + values, + Arc::new(BinaryViewArray::from_iter_values( + (0..LARGE_ROWS).map(|row| text(row).into_bytes()), + )), + ], + None, + ); + let maps = MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + offsets, + entries, + None, + false, + ); + let columns: Vec = vec![Arc::new(lists), Arc::new(large_lists), Arc::new(maps)]; + let schema = Arc::new(Schema::new( + columns + .iter() + .enumerate() + .map(|(index, column)| { + Field::new(format!("value{index}"), column.data_type().clone(), false) + }) + .collect::>(), + )); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + let (responses, status) = execute(BatchCase { + schema, + batches: vec![Batch::Arrow(batch)], + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(completed(&responses)); + let batches = output_batches(&responses); + assert_eq!(batches.len(), 1); + assert!( + assembled_batches(&responses, true)[0] + .num_chunks_in_batch + .unwrap() + > 1 + ); + let mut offset = 0; + for batch in batches { + let lists = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let large_lists = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let maps = batch.column(2).as_any().downcast_ref::().unwrap(); + for row in 0..batch.num_rows() { + let expected = text(offset + row); + let list = lists.value(row); + assert_eq!( + list.as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + let large_list = large_lists.value(row); + assert_eq!( + large_list + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + let entries = maps.value(row); + assert_eq!( + entries + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected + ); + assert_eq!( + entries + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + expected.as_bytes() + ); + } + offset += batch.num_rows(); + } + assert_eq!(offset, LARGE_ROWS); +} diff --git a/crates/spark-connect-server/tests/server_shutdown.rs b/crates/spark-connect-server/tests/server_shutdown.rs new file mode 100644 index 000000000..ac624c930 --- /dev/null +++ b/crates/spark-connect-server/tests/server_shutdown.rs @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use futures::Stream; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::Status; + +#[derive(Default)] +struct Spy { + streams_started: AtomicUsize, + streams_dropped: AtomicUsize, + configs_started: AtomicUsize, + configs_dropped: AtomicUsize, + sessions_dropped: AtomicUsize, +} + +struct PendingEngine(Arc); + +impl Engine for PendingEngine { + fn create_session(&self, _context: SessionContext) -> Result, Status> { + Ok(Arc::new(PendingSession(self.0.clone()))) + } +} + +struct PendingSession(Arc); + +impl Drop for PendingSession { + fn drop(&mut self) { + self.0.sessions_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +#[tonic::async_trait] +impl EngineSession for PendingSession { + async fn prepare(&self, _operation: proto::plan::OpType) -> Result { + Ok(PreparedOperation::Query(Box::new(PendingQuery( + self.0.clone(), + )))) + } + + async fn analyze( + &self, + _input: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented("shutdown fixture does not analyze")) + } + + async fn config( + &self, + _input: proto::config_request::operation::OpType, + ) -> Result { + let _call = PendingConfig(self.0.clone()); + self.0.configs_started.fetch_add(1, Ordering::SeqCst); + std::future::pending().await + } +} + +struct PendingConfig(Arc); + +impl Drop for PendingConfig { + fn drop(&mut self) { + self.0.configs_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +struct PendingQuery(Arc); + +#[tonic::async_trait] +impl PreparedQuery for PendingQuery { + fn schema(&self) -> SchemaRef { + Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])) + } + + async fn execute(self: Box) -> Result { + self.0.streams_started.fetch_add(1, Ordering::SeqCst); + Ok(Box::pin(PendingResults(self.0.clone()))) + } +} + +struct PendingResults(Arc); + +impl Stream for PendingResults { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } +} + +impl Drop for PendingResults { + fn drop(&mut self) { + self.0.streams_dropped.fetch_add(1, Ordering::SeqCst); + } +} + +async fn wait_for_count(counter: &AtomicUsize, expected: usize) { + tokio::time::timeout(Duration::from_secs(5), async { + while counter.load(Ordering::SeqCst) != expected { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("backend work must finish without disconnecting its client"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn aborting_serve_cancels_backend_work_before_clients_disconnect() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let spy = Arc::new(Spy::default()); + let server = tokio::spawn(spark_connect_server::serve_with_engine( + listener, + Arc::new(PendingEngine(spy.clone())), + std::future::pending(), + )); + let mut client = proto::spark_connect_service_client::SparkConnectServiceClient::connect( + format!("http://{address}"), + ) + .await + .unwrap(); + let session = uuid::Uuid::new_v4().to_string(); + let mut result = client + .execute_plan(proto::ExecutePlanRequest { + session_id: session.clone(), + operation_id: Some(uuid::Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(proto::Relation::default())), + }), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + assert!(result.message().await.unwrap().unwrap().schema.is_some()); + wait_for_count(&spy.streams_started, 1).await; + + let mut config_client = client.clone(); + let config = tokio::spawn(async move { + config_client + .config(proto::ConfigRequest { + session_id: session, + operation: Some(proto::config_request::Operation { + op_type: Some(proto::config_request::operation::OpType::GetAll( + Default::default(), + )), + }), + ..Default::default() + }) + .await + }); + wait_for_count(&spy.configs_started, 1).await; + server.abort(); + assert!(server.await.unwrap_err().is_cancelled()); + + // Retain both the streamed response and its client connection: cancellation + // must come from dropping serve_with_engine, not from a peer disconnect. + wait_for_count(&spy.streams_dropped, 1).await; + wait_for_count(&spy.configs_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert!(tokio::time::timeout(Duration::from_secs(5), config) + .await + .unwrap() + .unwrap() + .is_err()); + drop(result); + drop(client); +} diff --git a/crates/spark-connect-server/tests/support/arrow.rs b/crates/spark-connect-server/tests/support/arrow.rs new file mode 100644 index 000000000..c9fd69794 --- /dev/null +++ b/crates/spark-connect-server/tests/support/arrow.rs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Wire chunk negotiation, reassembly and IPC encoding shared by protocol tests. + +#![allow(dead_code)] + +use std::collections::HashSet; + +use arrow::ipc::writer::{IpcWriteOptions, StreamWriter}; +use arrow::ipc::CompressionType; +use arrow::record_batch::RecordBatch; +use prost::Message; +use proto::execute_plan_response::ResponseType; +use spark_connect_proto as proto; + +pub const DEFAULT_ARROW_CHUNK_SIZE: usize = 128 * 1024 * 1024 * 9 / 10; + +pub fn chunking(allow: bool, preferred: Option) -> proto::execute_plan_request::RequestOption { + proto::execute_plan_request::RequestOption { + request_option: Some( + proto::execute_plan_request::request_option::RequestOption::ResultChunkingOptions( + proto::ResultChunkingOptions { + allow_arrow_batch_chunking: allow, + preferred_arrow_chunk_size: preferred, + }, + ), + ), + } +} + +// Reassemble before Arrow decoding, just as Spark clients do. Every chunk counts +// the whole batch's rows and has its own response ID; row offsets advance once. +pub fn assembled_batches( + responses: &[proto::ExecutePlanResponse], + negotiated: bool, +) -> Vec { + let mut output = Vec::new(); + let mut pending: Option = None; + let (mut offset, mut index) = (0, 0); + let mut data = Vec::new(); + let mut response_ids = HashSet::new(); + for response in responses { + assert!(response_ids.insert(&response.response_id)); + let Some(ResponseType::ArrowBatch(chunk)) = &response.response_type else { + continue; + }; + // The negotiated limit covers IPC payload bytes, matching the JVM. The + // protobuf envelope adds identifiers, field tags and chunk metadata. + assert!(response.encoded_len() <= chunk.data.len() + 512); + if negotiated { + assert!(chunk.data.len() <= DEFAULT_ARROW_CHUNK_SIZE); + } + assert_eq!(chunk.start_offset, Some(offset)); + if !negotiated { + assert_eq!(chunk.chunk_index, None); + assert_eq!(chunk.num_chunks_in_batch, None); + offset += chunk.row_count; + output.push(chunk.clone()); + continue; + } + assert_eq!(chunk.chunk_index, Some(index)); + let count = chunk.num_chunks_in_batch.unwrap(); + assert!(count > index); + if let Some(batch) = &mut pending { + assert_eq!(batch.row_count, chunk.row_count); + assert_eq!(batch.num_chunks_in_batch, Some(count)); + } else { + pending = Some(chunk.clone()); + } + data.extend_from_slice(&chunk.data); + index += 1; + if index == count { + let mut batch = pending.take().unwrap(); + batch.data = std::mem::take(&mut data).into(); + offset += batch.row_count; + output.push(batch); + index = 0; + } + } + assert!(pending.is_none(), "complete must not truncate a batch"); + output +} + +pub fn encoded(batch: &RecordBatch, compression: Option) -> Vec { + let options = IpcWriteOptions::default() + .try_with_compression(compression) + .unwrap(); + let mut data = Vec::new(); + { + let mut writer = + StreamWriter::try_new_with_options(&mut data, batch.schema().as_ref(), options) + .unwrap(); + writer.write(batch).unwrap(); + writer.finish().unwrap(); + } + data +} diff --git a/crates/spark-connect-server/tests/support/mod.rs b/crates/spark-connect-server/tests/support/mod.rs new file mode 100644 index 000000000..6723e0aab --- /dev/null +++ b/crates/spark-connect-server/tests/support/mod.rs @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Explicit backend outputs for frontend protocol and schema tests. +//! +//! Range.end is only a fixture lookup key. This backend neither interprets SQL +//! nor implements Spark operators; each case supplies batches or a chosen error. +//! Test binaries use different subsets of the fixture constructors. + +#![allow(dead_code)] + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow::array::Int64Array; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::record_batch::RecordBatch; +use spark_connect_proto as proto; +use spark_connect_server::engine::{ + BatchStream, ConfigResult, Engine, EngineSession, PreparedOperation, PreparedQuery, + ResultBatch, SessionContext, +}; +use tonic::{Code, Status}; + +#[derive(Clone)] +pub enum Batch { + Arrow(RecordBatch), + Ipc(Vec, i64), +} + +#[derive(Clone)] +enum Output { + Query { + schema: SchemaRef, + batches: Vec, + error: Option, + }, + Rejected(Code), +} + +#[derive(Clone)] +pub struct FixtureEngine { + outputs: HashMap, +} + +impl Default for FixtureEngine { + fn default() -> Self { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = |start, end| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(start..end))], + ) + .unwrap() + }; + Self { + outputs: HashMap::new(), + } + .with_batches(0, schema.clone(), Vec::new()) + .with_batches(1, schema.clone(), vec![batch(0, 1)]) + .with_batches( + 25_000, + schema.clone(), + vec![batch(0, 8192), batch(8192, 16_384), batch(16_384, 25_000)], + ) + } +} + +impl FixtureEngine { + pub fn with_batches(self, selector: i64, schema: SchemaRef, batches: Vec) -> Self { + self.with_results( + selector, + schema, + batches.into_iter().map(Batch::Arrow).collect(), + ) + } + + pub fn with_results(mut self, selector: i64, schema: SchemaRef, batches: Vec) -> Self { + self.outputs.insert( + selector, + Output::Query { + schema, + batches, + error: None, + }, + ); + self + } + + pub fn reject(mut self, selector: i64, code: Code) -> Self { + self.outputs.insert(selector, Output::Rejected(code)); + self + } + + pub fn stream_error(mut self, selector: i64, code: Code) -> Self { + self.outputs.insert( + selector, + Output::Query { + schema: Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])), + batches: Vec::new(), + error: Some(code), + }, + ); + self + } +} + +impl Engine for FixtureEngine { + fn create_session(&self, _: SessionContext) -> Result, Status> { + Ok(Arc::new(self.clone())) + } +} + +#[tonic::async_trait] +impl EngineSession for FixtureEngine { + async fn prepare(&self, operation: proto::plan::OpType) -> Result { + let proto::plan::OpType::Root(relation) = operation else { + return Err(Status::unimplemented("fixture accepts root selectors only")); + }; + let Some(proto::relation::RelType::Range(selector)) = relation.rel_type else { + return Err(Status::unimplemented("fixture requires a range selector")); + }; + match self.outputs.get(&selector.end) { + Some(Output::Query { + schema, + batches, + error, + }) => Ok(PreparedOperation::Query(Box::new(FixtureQuery { + schema: schema.clone(), + batches: batches.clone(), + error: *error, + }))), + Some(Output::Rejected(code)) => Err(Status::new(*code, "prepared fixture error")), + None => Err(Status::unimplemented("unknown fixture selector")), + } + } + + async fn analyze( + &self, + _: proto::analyze_plan_request::Analyze, + ) -> Result { + Err(Status::unimplemented( + "fixture does not analyze query semantics", + )) + } + + async fn config( + &self, + _: proto::config_request::operation::OpType, + ) -> Result { + Err(Status::unimplemented( + "fixture does not configure query semantics", + )) + } +} + +struct FixtureQuery { + schema: SchemaRef, + batches: Vec, + error: Option, +} + +#[tonic::async_trait] +impl PreparedQuery for FixtureQuery { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + async fn execute(self: Box) -> Result { + let batches = self.batches.into_iter().map(|batch| { + Ok(match batch { + Batch::Arrow(batch) => ResultBatch::Arrow(batch), + Batch::Ipc(data, row_count) => ResultBatch::Ipc { data, row_count }, + }) + }); + let failure = self + .error + .into_iter() + .map(|code| Err(Status::new(code, "streamed fixture error"))); + Ok(Box::pin(futures::stream::iter(batches.chain(failure)))) + } +} diff --git a/crates/spark-connect-server/tests/support/requests.rs b/crates/spark-connect-server/tests/support/requests.rs new file mode 100644 index 000000000..343e02100 --- /dev/null +++ b/crates/spark-connect-server/tests/support/requests.rs @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared request envelopes and terminal-stream assertions. Individual test +//! binaries use different subsets of these helpers. + +#![allow(dead_code)] + +use std::time::Duration; + +use proto::execute_plan_response::ResponseType; +use spark_connect_proto as proto; +use tonic::Code; +use uuid::Uuid; + +pub fn user(id: &str) -> Option { + Some(proto::UserContext { + user_id: id.into(), + ..Default::default() + }) +} + +pub fn relation(end: i64) -> proto::Relation { + proto::Relation { + rel_type: Some(proto::relation::RelType::Range(proto::Range { + end, + step: 1, + ..Default::default() + })), + ..Default::default() + } +} + +pub fn request(session: &str, end: i64) -> proto::ExecutePlanRequest { + proto::ExecutePlanRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: Some(Uuid::new_v4().to_string()), + plan: Some(proto::Plan { + op_type: Some(proto::plan::OpType::Root(relation(end))), + }), + ..Default::default() + } +} + +pub fn reattachable_request(session: &str, end: i64) -> proto::ExecutePlanRequest { + let mut request = request(session, end); + request + .request_options + .push(proto::execute_plan_request::RequestOption { + request_option: Some( + proto::execute_plan_request::request_option::RequestOption::ReattachOptions( + proto::ReattachOptions { reattachable: true }, + ), + ), + }); + request +} + +pub fn config_request( + session: &str, + input: proto::config_request::operation::OpType, +) -> proto::ConfigRequest { + proto::ConfigRequest { + session_id: session.into(), + user_context: user("alice"), + operation: Some(proto::config_request::Operation { + op_type: Some(input), + }), + ..Default::default() + } +} + +pub fn release(session: &str, operation: &str) -> proto::ReleaseExecuteRequest { + proto::ReleaseExecuteRequest { + session_id: session.into(), + user_context: user("alice"), + operation_id: operation.into(), + release: Some(proto::release_execute_request::Release::ReleaseAll( + proto::release_execute_request::ReleaseAll {}, + )), + ..Default::default() + } +} + +pub fn release_session(session: &str) -> proto::ReleaseSessionRequest { + proto::ReleaseSessionRequest { + session_id: session.into(), + user_context: user("alice"), + ..Default::default() + } +} + +pub async fn assert_cancelled(stream: &mut tonic::Streaming) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match stream.message().await { + Err(status) => { + assert_eq!(status.code(), Code::Cancelled); + break; + } + Ok(Some(response)) => assert!(!matches!( + response.response_type, + Some(ResponseType::ResultComplete(_)) + )), + Ok(None) => panic!("cancelled execution must report cancellation"), + } + } + }) + .await + .expect("cancelled execution must terminate"); +} diff --git a/crates/spark-connect-server/tests/support/server.rs b/crates/spark-connect-server/tests/support/server.rs new file mode 100644 index 000000000..74602859e --- /dev/null +++ b/crates/spark-connect-server/tests/support/server.rs @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use spark_connect_proto::spark_connect_service_client::SparkConnectServiceClient; +use spark_connect_server::engine::Engine; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; +use tonic::transport::Channel; + +pub struct Server { + // Some test binaries need a second connection to the same listener. + #[allow(dead_code)] + pub address: SocketAddr, + pub client: SparkConnectServiceClient, + pub shutdown: Option>, + task: Option>, +} + +impl Server { + pub async fn start_with_engine(engine: Arc) -> Self { + Self::start_with_options(engine, spark_connect_server::ServerOptions::default()).await + } + + pub async fn start_with_options( + engine: Arc, + options: spark_connect_server::ServerOptions, + ) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (shutdown, stopped) = oneshot::channel(); + let task = tokio::spawn(async move { + spark_connect_server::serve_with_engine_and_options(listener, engine, options, async { + let _ = stopped.await; + }) + .await + .unwrap(); + }); + let client = SparkConnectServiceClient::connect(format!("http://{address}")) + .await + .unwrap(); + Self { + address, + client, + shutdown: Some(shutdown), + task: Some(task), + } + } + + pub async fn stop(mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap()) + .await + .unwrap() + .unwrap(); + } +} + +impl Drop for Server { + fn drop(&mut self) { + if let Some(shutdown) = self.shutdown.take() { + let _ = shutdown.send(()); + } + if let Some(task) = self.task.take() { + task.abort(); + } + } +} diff --git a/dev/design/ci.md b/dev/design/ci.md index 47ecc8a30..531267cb7 100644 --- a/dev/design/ci.md +++ b/dev/design/ci.md @@ -16,13 +16,15 @@ The CI consists of these main workflows: and macOS universal2 wheels using the release recipe 4. **Coverage** (`.github/workflows/coverage.yml`) - Rust and Python coverage, with regression floors +5. **Experimental server frontend** (`.github/workflows/server-frontend.yml`) - Test the + optional Rust protocol frontend with stable Rust and MSRV 1.88 The other workflows are `lint.yml` (rustfmt, clippy, rustdoc, ruff), `generated-drift.yml`, `vendor-drift.yml`, `docs.yml`, `pandas-parity.yml`, `pipelines-parity.yml` and `release.yml`. Most workflows run on pushes and pull requests targeting `master` and `branch-*`. Several also -accept manual dispatch. `docs.yml` and `wheel-build-checks.yml` run only when files matching their -path filters change. The release wheel build checks cover changes to packaged Python sources, +accept manual dispatch. `docs.yml`, `wheel-build-checks.yml` and `server-frontend.yml` filter push +and pull-request runs to matching paths. The release wheel build checks cover packaged Python sources, protocol definitions, dependencies and build configuration. `release.yml` runs on a schedule and on manual dispatch. @@ -69,6 +71,18 @@ The same workflow has four more jobs: This job should always pass; failures indicate incomplete implementation or test regressions. +## Experimental server frontend (`server-frontend.yml`) + +This focused workflow runs on frontend, protocol, workspace manifest, +lockfile, Cargo configuration, toolchain and workflow changes. Each stable Rust and Rust +1.88.0 job has a 45-minute limit, runs `cargo test --locked -p spark-connect-server`, +and checks the protocol crate with its `server` feature both off and on. +The existing workspace lint job covers the frontend with its pinned Rust toolchain; +test jobs do not promote warnings from a floating or older compiler to errors. +Ubuntu 24.04 supplies a protobuf compiler with +proto3 optional support. Tests use injected engine fixtures and require no execution backend; +they cover frontend protocol behavior rather than Spark execution compatibility. + ## Python Connect Parity (`build_python_connect.yml`) **Status**: Gating. The parity and drop-in jobs pass only when every test that runs passes, except From 5d3ca37757b6711930b6b16dce01b29344373813 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 20 Nov 2026 12:48:25 +0900 Subject: [PATCH 2/3] Cover the frontend's bounded shutdown drain, shutting-down session lookup and negative IPC row counts Address the three comments of the review https://github.com/apache/spark-connect-rust/pull/870#pullrequestreview-5986086240: - shutdown_releases_backend_session_while_a_result_remains_unread keeps its unread 1 KiB-window peer connected through server.stop(), so serve must return through the two-second transport drain bound of lib.rs. - requests_handled_after_shutdown_begins_do_not_create_backend_sessions sends an ExecutePlan's headers before the shutdown signal and its body after, for a session no request used, and expects UNAVAILABLE with no factory call. - negative_engine_ipc_row_counts_fail_before_forwarding passes an engine IPC batch with row_count -1 to send_batch and expects INTERNAL, an unchanged offset and output schema, and no response. Each test fails when the guard it covers is removed. Co-authored-by: Isaac --- crates/spark-connect-server/src/service.rs | 37 +++++++++++ .../tests/engine_injection.rs | 66 +++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index ce5ddd322..07f96b094 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -1000,6 +1000,43 @@ mod tests { } } + #[tokio::test] + async fn negative_engine_ipc_row_counts_fail_before_forwarding() { + use arrow::array::Int64Array; + use arrow::datatypes::{DataType, Field, Schema}; + + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![1]))]) + .unwrap(); + let (data, _) = encode_batch(&batch).unwrap(); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("negative_row_count").unwrap(); + let (sender, mut receiver) = mpsc::channel(2); + let mut offset = 7; + let mut output_schema = None; + let error = send_batch( + &sender, + "session", + "negative_row_count", + ResultBatch::Ipc { + data, + row_count: -1, + }, + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(CHUNK_SIZE), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::Internal); + assert_eq!(error.message(), "negative engine IPC row count"); + assert_eq!(offset, 7); + assert!(output_schema.is_none()); + assert!(receiver.try_recv().is_err()); + } + #[tokio::test] async fn overflowing_engine_batches_do_not_partially_emit() { use arrow::array::LargeStringArray; diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs index a7dd12657..46388dcf3 100644 --- a/crates/spark-connect-server/tests/engine_injection.rs +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -1221,6 +1221,72 @@ async fn shutdown_releases_backend_session_while_a_result_remains_unread() { wait_for_count(&spy.streams_dropped, 1).await; wait_for_count(&spy.sessions_dropped, 1).await; assert!(!response.body().is_end_stream()); + // The peer stays connected without reading, so only the bounded transport + // drain lets serve return. Disconnect it only afterwards. + tokio::time::timeout(Duration::from_secs(4), server.stop()) + .await + .expect("serve must return within its bounded drain while the peer stays connected"); + drop(response); + drop(upload); + drop(client); + connection.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn requests_handled_after_shutdown_begins_do_not_create_backend_sessions() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + // The cancellation of this execution shows that shutdown has closed the frontend. + let live_session = Uuid::new_v4().to_string(); + let mut live = start_live(&mut server, reattachable_request(&live_session, LIVE)).await; + let (mut client, mut connection) = h2::client::handshake( + tokio::net::TcpStream::connect(server.address) + .await + .unwrap(), + ) + .await + .unwrap(); + let mut ping_pong = connection.ping_pong().unwrap(); + let connection = tokio::spawn(async move { + let _ = connection.await; + }); + // A session that no request used before shutdown. + let body = request(&Uuid::new_v4().to_string(), 1).encode_to_vec(); + let mut payload = vec![0]; + payload.extend_from_slice(&(body.len() as u32).to_be_bytes()); + payload.extend_from_slice(&body); + let request = tonic::codegen::http::Request::builder() + .method("POST") + .uri(format!( + "http://{}/spark.connect.SparkConnectService/ExecutePlan", + server.address + )) + .header("content-type", "application/grpc") + .body(()) + .unwrap(); + let (response, mut upload) = client.send_request(request, false).unwrap(); + // h2 can write the first PING ahead of the request headers; the second follows + // them, so its PONG shows that the server received the headers before shutdown. + for _ in 0..2 { + ping_pong.ping(h2::Ping::opaque()).await.unwrap(); + } + server.shutdown.take().unwrap().send(()).unwrap(); + assert_cancelled(&mut live).await; + // The handler runs once the body arrives, after shutdown has begun. + upload.send_data(payload.into(), true).unwrap(); + let response = tokio::time::timeout(Duration::from_secs(5), response) + .await + .expect("the request must be answered before the transport drain ends") + .unwrap(); + let status = Status::from_header_map(response.headers()) + .expect("the frontend must reject the request before it streams a result"); + assert_eq!(status.code(), Code::Unavailable); + assert_eq!(status.message(), "server is shutting down"); + assert_eq!( + *spy.factory_calls.lock().unwrap(), + vec![("alice".to_string(), live_session)] + ); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); drop(response); drop(upload); drop(client); From ba2db67438ccb143eeec2261f0199de2d834f304 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Fri, 20 Nov 2026 12:52:48 +0900 Subject: [PATCH 3/3] Regenerate the JNI dependency inventory for the workspace Cargo.toml note crates/spark-connect-server/jvm/native-licenses/dependencies.json records the SHA-256 of the workspace Cargo.toml, which this branch changes with its note on tonic's server features. The JNI dependency closure is unchanged. Co-authored-by: Isaac --- .../spark-connect-server/jvm/native-licenses/dependencies.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/spark-connect-server/jvm/native-licenses/dependencies.json b/crates/spark-connect-server/jvm/native-licenses/dependencies.json index 4e1471886..1f6c94530 100644 --- a/crates/spark-connect-server/jvm/native-licenses/dependencies.json +++ b/crates/spark-connect-server/jvm/native-licenses/dependencies.json @@ -1,7 +1,7 @@ { "inputs": { "Cargo.lock": "4e72bf09e4de3cfaf0b7c996d35b9d788bc134b2c93e522579a322d8d6500d5c", - "Cargo.toml": "7643b23addd58bdbeecd687297526c3ddc8c3b85f0e019d9ef334bf8c0e3dbb6", + "Cargo.toml": "c6633f9ffab4f49000d4e4ca9b7496b6b3025ca711170970c7c74df16980a996", "LICENSE": "0d1dda3ddbb607c0ce754eddb23aa9b25b96f4f937b79b32605614c9ff1bfb96", "NOTICE": "f34ebcb547d17c3d7aab2d024c4685d4b17d49182480727b70cad2eccfc35274", "crates/spark-connect-proto/Cargo.toml": "5b43301089f8496478820fa2b063e544c2255da7cf8f1a33240ea915f2fe28ec",