Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ should_implement_trait = "allow"
[workspace.dependencies]
# gRPC / protobuf. The client of tonic only, its channel and generated code: the crates that test
# against a gRPC server of their own enable its `server` and `router` in their dev-dependencies.
# The unpublished spark-connect-server enables them in its normal dependencies.
tonic = { version = "0.14", default-features = false, features = ["channel", "codegen"] }
tonic-prost = "0.14"
tonic-prost-build = "0.14"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"inputs": {
"Cargo.lock": "4e72bf09e4de3cfaf0b7c996d35b9d788bc134b2c93e522579a322d8d6500d5c",
"Cargo.toml": "7643b23addd58bdbeecd687297526c3ddc8c3b85f0e019d9ef334bf8c0e3dbb6",
"Cargo.toml": "c6633f9ffab4f49000d4e4ca9b7496b6b3025ca711170970c7c74df16980a996",
"LICENSE": "0d1dda3ddbb607c0ce754eddb23aa9b25b96f4f937b79b32605614c9ff1bfb96",
"NOTICE": "f34ebcb547d17c3d7aab2d024c4685d4b17d49182480727b70cad2eccfc35274",
"crates/spark-connect-proto/Cargo.toml": "5b43301089f8496478820fa2b063e544c2255da7cf8f1a33240ea915f2fe28ec",
Expand Down
37 changes: 37 additions & 0 deletions crates/spark-connect-server/src/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1000,6 +1000,43 @@ mod tests {
}
}

#[tokio::test]
async fn negative_engine_ipc_row_counts_fail_before_forwarding() {
use arrow::array::Int64Array;
use arrow::datatypes::{DataType, Field, Schema};

let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![1]))])
.unwrap();
let (data, _) = encode_batch(&batch).unwrap();
let session = Arc::new(Session::new(Arc::new(RegistrySession)));
let (guard, _) = session.operation("negative_row_count").unwrap();
let (sender, mut receiver) = mpsc::channel(2);
let mut offset = 7;
let mut output_schema = None;
let error = send_batch(
&sender,
"session",
"negative_row_count",
ResultBatch::Ipc {
data,
row_count: -1,
},
schema.as_ref(),
&mut offset,
&mut output_schema,
&guard,
Some(CHUNK_SIZE),
)
.await
.unwrap_err();
assert_eq!(error.code(), tonic::Code::Internal);
assert_eq!(error.message(), "negative engine IPC row count");
assert_eq!(offset, 7);
assert!(output_schema.is_none());
assert!(receiver.try_recv().is_err());
}

#[tokio::test]
async fn overflowing_engine_batches_do_not_partially_emit() {
use arrow::array::LargeStringArray;
Expand Down
129 changes: 129 additions & 0 deletions crates/spark-connect-server/tests/engine_injection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1221,6 +1221,72 @@ async fn shutdown_releases_backend_session_while_a_result_remains_unread() {
wait_for_count(&spy.streams_dropped, 1).await;
wait_for_count(&spy.sessions_dropped, 1).await;
assert!(!response.body().is_end_stream());
// The peer stays connected without reading, so only the bounded transport
// drain lets serve return. Disconnect it only afterwards.
tokio::time::timeout(Duration::from_secs(4), server.stop())
.await
.expect("serve must return within its bounded drain while the peer stays connected");
drop(response);
drop(upload);
drop(client);
connection.abort();
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn requests_handled_after_shutdown_begins_do_not_create_backend_sessions() {
let spy = Arc::new(Spy::default());
let mut server = start_server(spy.clone()).await;
// The cancellation of this execution shows that shutdown has closed the frontend.
let live_session = Uuid::new_v4().to_string();
let mut live = start_live(&mut server, reattachable_request(&live_session, LIVE)).await;
let (mut client, mut connection) = h2::client::handshake(
tokio::net::TcpStream::connect(server.address)
.await
.unwrap(),
)
.await
.unwrap();
let mut ping_pong = connection.ping_pong().unwrap();
let connection = tokio::spawn(async move {
let _ = connection.await;
});
// A session that no request used before shutdown.
let body = request(&Uuid::new_v4().to_string(), 1).encode_to_vec();
let mut payload = vec![0];
payload.extend_from_slice(&(body.len() as u32).to_be_bytes());
payload.extend_from_slice(&body);
let request = tonic::codegen::http::Request::builder()
.method("POST")
.uri(format!(
"http://{}/spark.connect.SparkConnectService/ExecutePlan",
server.address
))
.header("content-type", "application/grpc")
.body(())
.unwrap();
let (response, mut upload) = client.send_request(request, false).unwrap();
// h2 can write the first PING ahead of the request headers; the second follows
// them, so its PONG shows that the server received the headers before shutdown.
for _ in 0..2 {
ping_pong.ping(h2::Ping::opaque()).await.unwrap();
}
server.shutdown.take().unwrap().send(()).unwrap();
assert_cancelled(&mut live).await;
// The handler runs once the body arrives, after shutdown has begun.
upload.send_data(payload.into(), true).unwrap();
let response = tokio::time::timeout(Duration::from_secs(5), response)
.await
.expect("the request must be answered before the transport drain ends")
.unwrap();
let status = Status::from_header_map(response.headers())
.expect("the frontend must reject the request before it streams a result");
assert_eq!(status.code(), Code::Unavailable);
assert_eq!(status.message(), "server is shutting down");
assert_eq!(
*spy.factory_calls.lock().unwrap(),
vec![("alice".to_string(), live_session)]
);
assert_eq!(spy.prepares.load(Ordering::SeqCst), 1);
drop(response);
drop(upload);
drop(client);
Expand Down Expand Up @@ -1304,6 +1370,69 @@ async fn operation_reservation_precedes_prepare_and_cleans_up_after_cancellation
server.stop().await;
}

async fn start_blocked_prepare(
server: &Server,
spy: &Spy,
session: &str,
) -> tokio::task::JoinSet<Status> {
let blocked = request(session, BLOCKED_PREPARE);
let mut pending_client = server.client.clone();
// Abort the client task on a failed assertion rather than leaving an RPC running.
let mut pending = tokio::task::JoinSet::new();
pending.spawn(async move { pending_client.execute_plan(blocked).await.unwrap_err() });
wait_for_count(&spy.prepares, 1).await;
pending
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn session_release_cancels_pending_prepare_and_drops_backend_session() {
let spy = Arc::new(Spy::default());
let mut server = start_server(spy.clone()).await;
let session = Uuid::new_v4().to_string();
let mut pending = start_blocked_prepare(&server, &spy, &session).await;
server
.client
.release_session(release_session(&session))
.await
.unwrap();

let status = tokio::time::timeout(Duration::from_secs(5), pending.join_next())
.await
.unwrap()
.unwrap()
.unwrap();
assert_eq!(status.code(), Code::Cancelled);
assert_eq!(status.message(), "session released");
wait_for_count(&spy.blocked_prepares_dropped, 1).await;
wait_for_count(&spy.sessions_dropped, 1).await;
assert_eq!(spy.executions.load(Ordering::SeqCst), 0);
// Keep server.client connected until cancellation and backend teardown are proven.
server.stop().await;
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn shutdown_cancels_pending_prepare_before_transport_drain_expires() {
let spy = Arc::new(Spy::default());
let mut server = start_server(spy.clone()).await;
let session = Uuid::new_v4().to_string();
let mut pending = start_blocked_prepare(&server, &spy, &session).await;

// Cancellation must finish before the two-second transport drain can drop the RPC.
tokio::time::timeout(Duration::from_secs(1), async {
server.shutdown.take().unwrap().send(()).unwrap();
let status = pending.join_next().await.unwrap().unwrap();
assert_eq!(status.code(), Code::Cancelled);
assert_eq!(status.message(), "session released");
wait_for_count(&spy.blocked_prepares_dropped, 1).await;
wait_for_count(&spy.sessions_dropped, 1).await;
assert_eq!(spy.executions.load(Ordering::SeqCst), 0);
})
.await
.expect("pending preparation must cancel before the transport drain expires");
// Keep server.client connected until cancellation and backend teardown are proven.
server.stop().await;
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn release_and_shutdown_cancel_backend_metadata_futures_before_dropping_sessions() {
let spy = Arc::new(Spy::default());
Expand Down
74 changes: 74 additions & 0 deletions crates/spark-connect-server/tests/schema_consistency.rs
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,80 @@ fn views(rows: usize) -> RecordBatch {
.unwrap()
}

fn materialized_view() -> RecordBatch {
RecordBatch::try_new(
strings(DataType::LargeUtf8, false).schema(),
vec![Arc::new(LargeStringArray::from(vec![text(0)]))],
)
.unwrap()
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn materialized_views_can_mix_with_matching_arrow_and_ipc_batches() {
let view = views(1);
let large = strings(DataType::LargeUtf8, false);
let materialized = materialized_view();
let data = ipc(&large);
for use_ipc in [false, true] {
for view_first in [false, true] {
let other = if use_ipc {
Batch::Ipc(data.clone(), 1)
} else {
Batch::Arrow(large.clone())
};
let mut batches = vec![Batch::Arrow(view.clone()), other];
let mut expected = vec![materialized.clone(), large.clone()];
if !view_first {
batches.reverse();
expected.reverse();
}
let (responses, status) = execute(BatchCase {
schema: view.schema(),
batches,
})
.await;
assert!(status.is_none(), "{status:?}");
assert!(responses.last().is_some_and(|response| matches!(
response.response_type,
Some(proto::execute_plan_response::ResponseType::ResultComplete(
_
))
)));
assert_eq!(output_batches(&responses), expected);
if use_ipc {
let output = assembled_batches(&responses, true);
let ipc_index = if view_first { 1 } else { 0 };
assert_eq!(output[ipc_index].data.as_ref(), data.as_slice());
}
}
}
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn materialized_views_cannot_mix_with_preserved_view_ipc() {
let view = views(1);
let materialized = materialized_view();
let data = ipc(&view);
for view_first in [false, true] {
let mut batches = vec![Batch::Arrow(view.clone()), Batch::Ipc(data.clone(), 1)];
if !view_first {
batches.reverse();
}
let (responses, status) = execute(BatchCase {
schema: view.schema(),
batches,
})
.await;
assert_schema_error(&responses, status);
let expected = if view_first {
materialized.clone()
} else {
view.clone()
};
assert_eq!(output_batches(&responses), vec![expected]);
}
}

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn view_ipc_remains_unchanged_for_small_and_oversized_batches() {
for rows in [2, LARGE_ROWS] {
Expand Down
Loading