diff --git a/Cargo.toml b/Cargo.toml index 70f41219..63d6821d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,6 +92,7 @@ should_implement_trait = "allow" [workspace.dependencies] # gRPC / protobuf. The client of tonic only, its channel and generated code: the crates that test # against a gRPC server of their own enable its `server` and `router` in their dev-dependencies. +# The unpublished spark-connect-server enables them in its normal dependencies. tonic = { version = "0.14", default-features = false, features = ["channel", "codegen"] } tonic-prost = "0.14" tonic-prost-build = "0.14" diff --git a/crates/spark-connect-server/jvm/native-licenses/dependencies.json b/crates/spark-connect-server/jvm/native-licenses/dependencies.json index 4e147188..1f6c9453 100644 --- a/crates/spark-connect-server/jvm/native-licenses/dependencies.json +++ b/crates/spark-connect-server/jvm/native-licenses/dependencies.json @@ -1,7 +1,7 @@ { "inputs": { "Cargo.lock": "4e72bf09e4de3cfaf0b7c996d35b9d788bc134b2c93e522579a322d8d6500d5c", - "Cargo.toml": "7643b23addd58bdbeecd687297526c3ddc8c3b85f0e019d9ef334bf8c0e3dbb6", + "Cargo.toml": "c6633f9ffab4f49000d4e4ca9b7496b6b3025ca711170970c7c74df16980a996", "LICENSE": "0d1dda3ddbb607c0ce754eddb23aa9b25b96f4f937b79b32605614c9ff1bfb96", "NOTICE": "f34ebcb547d17c3d7aab2d024c4685d4b17d49182480727b70cad2eccfc35274", "crates/spark-connect-proto/Cargo.toml": "5b43301089f8496478820fa2b063e544c2255da7cf8f1a33240ea915f2fe28ec", diff --git a/crates/spark-connect-server/src/service.rs b/crates/spark-connect-server/src/service.rs index ce5ddd32..07f96b09 100644 --- a/crates/spark-connect-server/src/service.rs +++ b/crates/spark-connect-server/src/service.rs @@ -1000,6 +1000,43 @@ mod tests { } } + #[tokio::test] + async fn negative_engine_ipc_row_counts_fail_before_forwarding() { + use arrow::array::Int64Array; + use arrow::datatypes::{DataType, Field, Schema}; + + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![1]))]) + .unwrap(); + let (data, _) = encode_batch(&batch).unwrap(); + let session = Arc::new(Session::new(Arc::new(RegistrySession))); + let (guard, _) = session.operation("negative_row_count").unwrap(); + let (sender, mut receiver) = mpsc::channel(2); + let mut offset = 7; + let mut output_schema = None; + let error = send_batch( + &sender, + "session", + "negative_row_count", + ResultBatch::Ipc { + data, + row_count: -1, + }, + schema.as_ref(), + &mut offset, + &mut output_schema, + &guard, + Some(CHUNK_SIZE), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::Internal); + assert_eq!(error.message(), "negative engine IPC row count"); + assert_eq!(offset, 7); + assert!(output_schema.is_none()); + assert!(receiver.try_recv().is_err()); + } + #[tokio::test] async fn overflowing_engine_batches_do_not_partially_emit() { use arrow::array::LargeStringArray; diff --git a/crates/spark-connect-server/tests/engine_injection.rs b/crates/spark-connect-server/tests/engine_injection.rs index 6000f525..46388dcf 100644 --- a/crates/spark-connect-server/tests/engine_injection.rs +++ b/crates/spark-connect-server/tests/engine_injection.rs @@ -1221,6 +1221,72 @@ async fn shutdown_releases_backend_session_while_a_result_remains_unread() { wait_for_count(&spy.streams_dropped, 1).await; wait_for_count(&spy.sessions_dropped, 1).await; assert!(!response.body().is_end_stream()); + // The peer stays connected without reading, so only the bounded transport + // drain lets serve return. Disconnect it only afterwards. + tokio::time::timeout(Duration::from_secs(4), server.stop()) + .await + .expect("serve must return within its bounded drain while the peer stays connected"); + drop(response); + drop(upload); + drop(client); + connection.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn requests_handled_after_shutdown_begins_do_not_create_backend_sessions() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + // The cancellation of this execution shows that shutdown has closed the frontend. + let live_session = Uuid::new_v4().to_string(); + let mut live = start_live(&mut server, reattachable_request(&live_session, LIVE)).await; + let (mut client, mut connection) = h2::client::handshake( + tokio::net::TcpStream::connect(server.address) + .await + .unwrap(), + ) + .await + .unwrap(); + let mut ping_pong = connection.ping_pong().unwrap(); + let connection = tokio::spawn(async move { + let _ = connection.await; + }); + // A session that no request used before shutdown. + let body = request(&Uuid::new_v4().to_string(), 1).encode_to_vec(); + let mut payload = vec![0]; + payload.extend_from_slice(&(body.len() as u32).to_be_bytes()); + payload.extend_from_slice(&body); + let request = tonic::codegen::http::Request::builder() + .method("POST") + .uri(format!( + "http://{}/spark.connect.SparkConnectService/ExecutePlan", + server.address + )) + .header("content-type", "application/grpc") + .body(()) + .unwrap(); + let (response, mut upload) = client.send_request(request, false).unwrap(); + // h2 can write the first PING ahead of the request headers; the second follows + // them, so its PONG shows that the server received the headers before shutdown. + for _ in 0..2 { + ping_pong.ping(h2::Ping::opaque()).await.unwrap(); + } + server.shutdown.take().unwrap().send(()).unwrap(); + assert_cancelled(&mut live).await; + // The handler runs once the body arrives, after shutdown has begun. + upload.send_data(payload.into(), true).unwrap(); + let response = tokio::time::timeout(Duration::from_secs(5), response) + .await + .expect("the request must be answered before the transport drain ends") + .unwrap(); + let status = Status::from_header_map(response.headers()) + .expect("the frontend must reject the request before it streams a result"); + assert_eq!(status.code(), Code::Unavailable); + assert_eq!(status.message(), "server is shutting down"); + assert_eq!( + *spy.factory_calls.lock().unwrap(), + vec![("alice".to_string(), live_session)] + ); + assert_eq!(spy.prepares.load(Ordering::SeqCst), 1); drop(response); drop(upload); drop(client); @@ -1304,6 +1370,69 @@ async fn operation_reservation_precedes_prepare_and_cleans_up_after_cancellation server.stop().await; } +async fn start_blocked_prepare( + server: &Server, + spy: &Spy, + session: &str, +) -> tokio::task::JoinSet { + let blocked = request(session, BLOCKED_PREPARE); + let mut pending_client = server.client.clone(); + // Abort the client task on a failed assertion rather than leaving an RPC running. + let mut pending = tokio::task::JoinSet::new(); + pending.spawn(async move { pending_client.execute_plan(blocked).await.unwrap_err() }); + wait_for_count(&spy.prepares, 1).await; + pending +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn session_release_cancels_pending_prepare_and_drops_backend_session() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut pending = start_blocked_prepare(&server, &spy, &session).await; + server + .client + .release_session(release_session(&session)) + .await + .unwrap(); + + let status = tokio::time::timeout(Duration::from_secs(5), pending.join_next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(status.code(), Code::Cancelled); + assert_eq!(status.message(), "session released"); + wait_for_count(&spy.blocked_prepares_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!(spy.executions.load(Ordering::SeqCst), 0); + // Keep server.client connected until cancellation and backend teardown are proven. + server.stop().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_cancels_pending_prepare_before_transport_drain_expires() { + let spy = Arc::new(Spy::default()); + let mut server = start_server(spy.clone()).await; + let session = Uuid::new_v4().to_string(); + let mut pending = start_blocked_prepare(&server, &spy, &session).await; + + // Cancellation must finish before the two-second transport drain can drop the RPC. + tokio::time::timeout(Duration::from_secs(1), async { + server.shutdown.take().unwrap().send(()).unwrap(); + let status = pending.join_next().await.unwrap().unwrap(); + assert_eq!(status.code(), Code::Cancelled); + assert_eq!(status.message(), "session released"); + wait_for_count(&spy.blocked_prepares_dropped, 1).await; + wait_for_count(&spy.sessions_dropped, 1).await; + assert_eq!(spy.executions.load(Ordering::SeqCst), 0); + }) + .await + .expect("pending preparation must cancel before the transport drain expires"); + // Keep server.client connected until cancellation and backend teardown are proven. + server.stop().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn release_and_shutdown_cancel_backend_metadata_futures_before_dropping_sessions() { let spy = Arc::new(Spy::default()); diff --git a/crates/spark-connect-server/tests/schema_consistency.rs b/crates/spark-connect-server/tests/schema_consistency.rs index 2a3845f2..023c185f 100644 --- a/crates/spark-connect-server/tests/schema_consistency.rs +++ b/crates/spark-connect-server/tests/schema_consistency.rs @@ -241,6 +241,80 @@ fn views(rows: usize) -> RecordBatch { .unwrap() } +fn materialized_view() -> RecordBatch { + RecordBatch::try_new( + strings(DataType::LargeUtf8, false).schema(), + vec![Arc::new(LargeStringArray::from(vec![text(0)]))], + ) + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn materialized_views_can_mix_with_matching_arrow_and_ipc_batches() { + let view = views(1); + let large = strings(DataType::LargeUtf8, false); + let materialized = materialized_view(); + let data = ipc(&large); + for use_ipc in [false, true] { + for view_first in [false, true] { + let other = if use_ipc { + Batch::Ipc(data.clone(), 1) + } else { + Batch::Arrow(large.clone()) + }; + let mut batches = vec![Batch::Arrow(view.clone()), other]; + let mut expected = vec![materialized.clone(), large.clone()]; + if !view_first { + batches.reverse(); + expected.reverse(); + } + let (responses, status) = execute(BatchCase { + schema: view.schema(), + batches, + }) + .await; + assert!(status.is_none(), "{status:?}"); + assert!(responses.last().is_some_and(|response| matches!( + response.response_type, + Some(proto::execute_plan_response::ResponseType::ResultComplete( + _ + )) + ))); + assert_eq!(output_batches(&responses), expected); + if use_ipc { + let output = assembled_batches(&responses, true); + let ipc_index = if view_first { 1 } else { 0 }; + assert_eq!(output[ipc_index].data.as_ref(), data.as_slice()); + } + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn materialized_views_cannot_mix_with_preserved_view_ipc() { + let view = views(1); + let materialized = materialized_view(); + let data = ipc(&view); + for view_first in [false, true] { + let mut batches = vec![Batch::Arrow(view.clone()), Batch::Ipc(data.clone(), 1)]; + if !view_first { + batches.reverse(); + } + let (responses, status) = execute(BatchCase { + schema: view.schema(), + batches, + }) + .await; + assert_schema_error(&responses, status); + let expected = if view_first { + materialized.clone() + } else { + view.clone() + }; + assert_eq!(output_batches(&responses), vec![expected]); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn view_ipc_remains_unchanged_for_small_and_oversized_batches() { for rows in [2, LARGE_ROWS] {