Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/workflows/native-server.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ jobs:
timeout-minutes: 75
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false

- name: Install Rust MSRV
# Use runner-provided rustup to keep actions limited to actions/*.
Expand Down Expand Up @@ -143,6 +145,9 @@ jobs:
- name: Test native JAR packaging
run: python3 -B crates/spark-connect-server/jvm/test_package_native.py

- name: Test native loader runner guards
run: python3 -B crates/spark-connect-server/jvm/test_native_loader_runner.py

- name: Test C bridge
run: crates/spark-connect-server/jni/test-c-bridge.sh

Expand Down Expand Up @@ -287,6 +292,8 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false

- uses: actions/setup-python@v5
with:
Expand Down Expand Up @@ -354,6 +361,8 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false

- uses: actions/setup-java@v4
with:
Expand Down
1 change: 1 addition & 0 deletions crates/spark-connect-server/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -672,6 +672,7 @@ cargo clippy -p spark-connect-server -p spark-connect-jni --all-targets --no-dep
crates/spark-connect-server/jni/test-c-bridge.sh
crates/spark-connect-server/jni/test-c-bridge.sh --sanitize
python3 -B crates/spark-connect-server/jvm/test_acceptance_guards.py
python3 -B crates/spark-connect-server/jvm/test_native_loader_runner.py
python3 -B crates/spark-connect-server/jvm/test_package_native.py
crates/spark-connect-server/jvm/test-native-loader.sh
```
Expand Down
19 changes: 17 additions & 2 deletions crates/spark-connect-server/jvm/test-native-loader.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,27 @@ esac
-I "$JAVA_HOME/include" -I "$JAVA_HOME/include/$JNI_PLATFORM" \
"$BUILD_DIR/fixture.c" -o "$LIBRARY"
TEST_CLASS=org.apache.spark.sql.connect.nativebridge.engine.NativeLibraryLoaderTest
"$JAVA_HOME/bin/java" -Xcheck:jni "-Djava.io.tmpdir=$BUILD_DIR/tmp" \
run_loader_test() {
local name="$1" marker="$2"
shift 2
local log="$BUILD_DIR/$name.log"
"$JAVA_HOME/bin/java" -Xcheck:jni "-Djava.io.tmpdir=$BUILD_DIR/tmp" \
"$@" 2>&1 | tee "$log" || return
if grep -Eq 'WARNING in native method|WARNING: JNI local refs:|FATAL ERROR in native method' "$log"; then
echo "native loader $name triggered a JNI diagnostic" >&2
return 1
fi
if ! grep -Eq "$marker" "$log"; then
echo "native loader $name success marker missing" >&2
return 1
fi
}
run_loader_test fixture '^NATIVE_LIBRARY_LOADER_OK checks=[1-9][0-9]*$' \
-cp "$BUILD_DIR/classes" "$TEST_CLASS" "$LIBRARY"
if [[ $# -eq 2 ]]; then
NATIVE_JAR="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")"
[[ -f "$NATIVE_JAR" ]] || { echo "native JAR not found: $NATIVE_JAR" >&2; exit 1; }
"$JAVA_HOME/bin/java" -Xcheck:jni "-Djava.io.tmpdir=$BUILD_DIR/tmp" \
run_loader_test packaged '^NATIVE_PACKAGED_LOAD_OK$' \
-cp "$BUILD_DIR/classes:$NATIVE_JAR" "$TEST_CLASS" --load-packaged
fi
if [[ -n "$(find "$BUILD_DIR/tmp" -mindepth 1 -print -quit)" ]]; then
Expand Down
159 changes: 159 additions & 0 deletions crates/spark-connect-server/jvm/test_native_loader_runner.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

"""Exercise loader runner output guards with fake tools, without compiling JNI."""

import os
from pathlib import Path
import shlex
import subprocess
import sys
import tempfile
import unittest


RUNNER = Path(__file__).with_name("test-native-loader.sh")
MARKERS = {
"fixture": "NATIVE_LIBRARY_LOADER_OK checks=1\n",
"packaged": "NATIVE_PACKAGED_LOAD_OK\n",
}


class NativeLoaderRunnerTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.directory = Path(temporary.name)
self.java_home = self.directory / "jdk"
self.tools = self.directory / "tools"
self.tools.mkdir()
self.executable(self.java_home / "bin/javac", "raise SystemExit(0)\n")
self.executable(self.java_home / "bin/java", """
import os
import sys

mode = "PACKAGED" if sys.argv[-1] == "--load-packaged" else "FIXTURE"
sys.stdout.write(os.environ["FAKE_" + mode + "_STDOUT"])
sys.stderr.write(os.environ.get("FAKE_" + mode + "_STDERR", ""))
raise SystemExit(int(os.environ.get("FAKE_" + mode + "_STATUS", "0")))
""")
self.executable(self.tools / "cc", "raise SystemExit(0)\n")
self.executable(self.tools / "version", "print('0.1.0')\n")
self.native_jar = self.directory / "native.jar"
self.native_jar.touch()

@staticmethod
def executable(path, source, interpreter=sys.executable):
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("#!/bin/sh\nexec " + shlex.quote(str(interpreter))
+ " -c " + shlex.quote(source) + ' "$@"\n', encoding="utf-8")
path.chmod(0o755)

def run_loader(self, mode, output, *, stderr="", status=0):
environment = os.environ.copy()
environment.update({
"JAVA_HOME": str(self.java_home),
"CC": str(self.tools / "cc"),
"NATIVE_BUILD_PYTHON": str(self.tools / "version"),
"TMPDIR": str(self.directory),
"PATH": str(self.tools) + os.pathsep + environment.get("PATH", os.defpath),
})
for name, marker in MARKERS.items():
environment[f"FAKE_{name.upper()}_STDOUT"] = marker
environment[f"FAKE_{mode.upper()}_STDOUT"] = output
environment[f"FAKE_{mode.upper()}_STDERR"] = stderr
environment[f"FAKE_{mode.upper()}_STATUS"] = str(status)
command = ["bash", str(RUNNER)]
if mode == "packaged":
command.extend(["--native-jar", str(self.native_jar)])
return subprocess.run(command, env=environment, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True, timeout=10)

def test_each_mode_requires_its_own_success_marker(self):
for mode, marker in MARKERS.items():
with self.subTest(mode=mode):
result = self.run_loader(mode, marker)
self.assertEqual(result.returncode, 0, result.stdout)
self.assertIn(marker, result.stdout)

def test_python_interpreter_path_with_spaces(self):
interpreter = self.directory / "python with spaces"
interpreter.symlink_to(sys.executable)
self.executable(self.java_home / "bin/javac", "raise SystemExit(0)\n", interpreter)
result = self.run_loader("fixture", MARKERS["fixture"])
self.assertEqual(result.returncode, 0, result.stdout)

def test_jni_diagnostics_fail_even_with_success_marker_and_zero_exit(self):
for mode, marker in MARKERS.items():
for diagnostic in ["WARNING in native method", "WARNING: JNI local refs:",
"FATAL ERROR in native method"]:
with self.subTest(mode=mode, diagnostic=diagnostic):
result = self.run_loader(mode, marker, stderr=diagnostic + ": probe\n")
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("triggered a JNI diagnostic", result.stdout)
self.assertIn(diagnostic, result.stdout)

def test_empty_output_fails(self):
for mode in MARKERS:
with self.subTest(mode=mode):
result = self.run_loader(mode, "")
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("success marker missing", result.stdout)

def test_other_modes_marker_does_not_pass(self):
for mode in MARKERS:
other = "packaged" if mode == "fixture" else "fixture"
with self.subTest(mode=mode):
result = self.run_loader(mode, MARKERS[other])
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("success marker missing", result.stdout)

def test_incomplete_or_embedded_markers_do_not_pass(self):
for mode, marker in MARKERS.items():
outputs = [marker.strip()[:-1] + "\n", "prefix " + marker,
marker.rstrip() + " suffix\n"]
if mode == "fixture":
outputs.append("NATIVE_LIBRARY_LOADER_OK checks=0\n")
for output in outputs:
with self.subTest(mode=mode, output=output):
result = self.run_loader(mode, output)
self.assertNotEqual(result.returncode, 0, result.stdout)
self.assertIn("success marker missing", result.stdout)

def test_java_failure_propagates_through_logging_pipeline(self):
for mode, marker in MARKERS.items():
with self.subTest(mode=mode):
result = self.run_loader(mode, marker, status=17)
self.assertEqual(result.returncode, 17, result.stdout)

def test_logging_failure_does_not_pass(self):
self.executable(self.tools / "tee", """
from pathlib import Path
import sys

output = sys.stdin.read()
Path(sys.argv[1]).write_text(output)
sys.stdout.write(output)
raise SystemExit(23)
""")
result = self.run_loader("fixture", MARKERS["fixture"])
self.assertEqual(result.returncode, 23, result.stdout)


if __name__ == "__main__":
unittest.main()
4 changes: 2 additions & 2 deletions crates/spark-connect-server/jvm/test_package_native.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,15 @@
import importlib.util
import json
from pathlib import Path
import stat
import shutil
import stat
import struct
import subprocess
import sys
import tempfile
import unittest
from unittest.mock import patch
import warnings
import zipfile


Expand Down Expand Up @@ -399,7 +400,6 @@ def test_combine_rejects_unexpected_resource(self):

def test_duplicate_archive_entries_are_rejected(self):
jar = self.package("linux-x86_64")
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning)
with zipfile.ZipFile(jar, "a") as archive:
Expand Down
4 changes: 2 additions & 2 deletions scripts/tests/test_gen_native_binary_licenses.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from unittest.mock import Mock, patch

SCRIPT = Path(__file__).resolve().parents[1] / "gen_native_binary_licenses.py"
SPEC = importlib.util.spec_from_file_location("gen_native_binary_licenses", SCRIPT)
Expand Down Expand Up @@ -131,7 +131,7 @@ def test_all_four_targets_and_correct_root_are_required(self):
self.generate()

def test_cargo_tree_resolves_native_normal_edges_without_wheel_features(self):
result = unittest.mock.Mock(returncode=0, stdout=b"tree", stderr=b"")
result = Mock(returncode=0, stdout=b"tree", stderr=b"")
with patch.object(gen.subprocess, "run", return_value=result) as run:
gen.cargo_tree(self.root, gen.TARGETS[1])
args = run.call_args.args[0]
Expand Down
Loading