[LIVY-1041] Add Spark 4 support and make it the default build

.github/workflows/integration-tests.yaml

@@ -28,11 +28,14 @@ jobs:
     container: ghcr.io/${{ github.repository_owner }}/livy-ci:latest
     strategy:
       matrix:
-        maven_profile:
-        - "-Pscala-2.12 -Pspark3"
-        jdk_path:
-        - "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
-        - "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
+        include:
+          - maven_profile: "-Pscala-2.12 -Pspark3"
+            jdk_path: "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
+          - maven_profile: "-Pscala-2.12 -Pspark3"
+            jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
+          # spark4 is the default build; JDK 17+ required.
+          - maven_profile: ""
+            jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
     steps:
     -
       name: Checkout

.github/workflows/unit-tests.yaml

@@ -29,11 +29,14 @@ jobs:
     container: ghcr.io/${{ github.repository_owner }}/livy-ci:latest
     strategy:
       matrix:
-        maven_profile:
-        - "-Pscala-2.12 -Pspark3"
-        jdk_path:
-        - "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
-        - "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
+        include:
+          - maven_profile: "-Pscala-2.12 -Pspark3"
+            jdk_path: "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
+          - maven_profile: "-Pscala-2.12 -Pspark3"
+            jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
+          # spark4 is the default build; JDK 17+ required.
+          - maven_profile: ""
+            jdk_path: "/usr/lib/jvm/java-17-openjdk-amd64/bin/java"
     steps:
     -
       name: Checkout

README.md

@@ -78,12 +78,12 @@ You can also use the provided [Dockerfile](./dev/docker/livy-dev-base/Dockerfile
 git clone https://github.com/apache/livy.git
 cd livy
 docker build -t livy-ci dev/docker/livy-dev-base/
-docker run --rm -it -v $(pwd):/workspace -v $HOME/.m2:/root/.m2 livy-ci mvn package -Pspark3 -Pscala-2.12
+docker run --rm -it -v $(pwd):/workspace -v $HOME/.m2:/root/.m2 livy-ci mvn package
 ```
 
 > **Note**: The `docker run` command maps the maven repository to your host machine's maven cache so subsequent runs will not need to download dependencies.
 
-By default Livy is built against Apache Spark 3.3.4, but the version of Spark used when running
+By default Livy is built against Apache Spark 4.1.2 with Scala 2.13, but the version of Spark used when running
 Livy does not need to match the version used to build Livy. Livy internally handles the differences
 between different Spark versions.
 
@@ -92,9 +92,19 @@ version of Spark without needing to rebuild.
 
 ### Build Profiles
 
-| Flag           | Purpose                                    |
-|----------------|--------------------------------------------|
-| -Phadoop2      | Choose Hadoop2 based build dependencies    |
-| -Pthriftserver | Build and test Livy Thrift Server modules  |
-| -Pspark3       | Choose Spark 3.x based build dependencies  |
-| -Pscala-2.12   | Choose Scala 2.12 based build dependencies |
+| Flag           | Purpose                                                                            |
+|----------------|------------------------------------------------------------------------------------|
+| -Phadoop2      | Choose Hadoop2 based build dependencies                                            |
+| -Pthriftserver | Build and test Livy Thrift Server modules                                          |
+| -Pspark3       | Choose Spark 3.x based build dependencies (use with `-Pscala-2.12`)                |
+| -Pscala-2.12   | Choose Scala 2.12 based build dependencies (use with `-Pspark3`)                   |
+
+Example — build against Spark 3:
+
+```
+mvn package -Pspark3 -Pscala-2.12
+```
+
+> **Note**: The default build targets Spark 4.1.2 and requires JDK 17 or JDK 21 with Scala 2.13 and Hadoop 3.4.1.
+> JDK 8, JDK 11 and Scala 2.12 are not supported by Spark 4. Supported Python
+> versions for Spark 4.1 are 3.10 – 3.14.

core/scala-2.13/pom.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.livy</groupId>
+  <artifactId>livy-core_2.13</artifactId>
+  <version>1.0.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <parent>
+    <groupId>org.apache.livy</groupId>
+    <artifactId>livy-core-parent</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

dev/docker/livy-dev-base/Dockerfile

@@ -72,7 +72,7 @@ ENV PATH="$HOME/pyenv/shims:$HOME/pyenv/bin:$HOME/bin:$PATH"
 
 # Python 3.11 is chosen because it is in the officially supported range of
 # BOTH matrix profiles, per each Spark release's python/setup.py:
-#   * -Pspark4 (Spark 4.1.2): python_requires=">=3.10", classifiers list
+#   * default / Spark 4.1.2: python_requires=">=3.10", classifiers list
 #     3.10 / 3.11 / 3.12 / 3.13 / 3.14.
 #   * -Pspark3 (Spark 3.5.6): python_requires=">=3.8", classifiers list
 #     3.8 / 3.9 / 3.10 / 3.11.

integration-test/src/test/scala/org/apache/livy/test/InteractiveIT.scala

@@ -30,27 +30,46 @@ import org.apache.livy.rsc.RSCConf
 import org.apache.livy.sessions._
 import org.apache.livy.test.framework.{BaseIntegrationTestSuite, LivyRestClient}
 
-class InteractiveIT extends BaseIntegrationTestSuite {
+class InteractiveIT extends BaseIntegrationTestSuite with ScalaVersionAware {
   test("basic interactive session") {
     withNewSession(Spark) { s =>
       s.run("val sparkVersion = sc.version").result().left.foreach(info(_))
       s.run("val scalaVersion = util.Properties.versionString").result().left.foreach(info(_))
-      s.run("1+1").verifyResult("res0: Int = 2\n")
+      // Scala 2.13's REPL prints a `val ` prefix before result names
+      // (`val res0: Int = 2`) whereas Scala 2.12 prints just `res0: Int = 2`.
+      // Accept both forms so the test passes on either the default Scala 2.13
+      // build or -Pscala-2.12.
+      s.run("1+1").verifyResult(s"${optionalValPrefixRegex}res0: Int = 2\n")
 
       // Ignore the following line if running on a external cluster due to config differences
       // with the mini cluster
-      s.run("""sc.getConf.get("spark.executor.instances")""").verifyResult("res1: String = 1\n")
+      s.run("""sc.getConf.get("spark.executor.instances")""")
+        .verifyResult(s"${optionalValPrefixRegex}res1: String = 1\n")
 
+      // Spark 4 relocated SQLContext into the `org.apache.spark.sql.classic`
+      // package; Spark 3 keeps it directly under `org.apache.spark.sql`. Match
+      // either shape (`SQLContext` or `classic.SQLContext`) and tolerate the
+      // Scala 2.13 REPL's `val ` prefix as elsewhere in this test.
       s.run("val sql = spark.sqlContext").verifyResult(
-        ".*" + Pattern.quote(
-        "sql: org.apache.spark.sql.SQLContext = org.apache.spark.sql.SQLContext") + ".*")
-      s.run("abcde").verifyError(evalue = ".*?:[0-9]+: error: not found: value abcde.*")
+        ".*sql: org\\.apache\\.spark\\.sql\\.(?:classic\\.)?SQLContext = " +
+        "org\\.apache\\.spark\\.sql\\.(?:classic\\.)?SQLContext.*")
+      // Scala 2.12's REPL prefixes compile errors with a "<source>:<line>:"
+      // location marker (e.g. "<console>:12: error: not found: value abcde"),
+      // while Scala 2.13 drops the marker and just emits "error: not found:
+      // value abcde". Accept both.
+      s.run("abcde").verifyError(evalue = ".*(?:.*?:[0-9]+: )?error: not found: value abcde.*")
       s.run("throw new IllegalStateException()")
         .verifyError(evalue = ".*java\\.lang\\.IllegalStateException.*")
 
-      // Verify query submission
+      // Verify query submission. Spark 4's Scala 2.13 REPL surfaces the runtime
+      // class name (`org.apache.spark.sql.classic.DataFrame`) rather than the
+      // compile-time alias (`org.apache.spark.sql.DataFrame`) that Spark 3
+      // prints, and prefixes the identifier with `val ` like elsewhere in this
+      // test. Accept both shapes.
       s.run(s"""val df = spark.createDataFrame(Seq(("jerry", 20), ("michael", 21)))""")
-        .verifyResult(".*" + Pattern.quote("df: org.apache.spark.sql.DataFrame") + ".*")
+        .verifyResult(
+          s".*${optionalValPrefixRegex}df: " +
+            "org\\.apache\\.spark\\.sql\\.(?:classic\\.)?DataFrame.*")
       s.run("df.createOrReplaceTempView(\"people\")").result()
       s.run("SELECT * FROM people", Some(SQL)).verifyResult(".*\"jerry\",20.*\"michael\",21.*")
 
@@ -167,9 +186,15 @@ class InteractiveIT extends BaseIntegrationTestSuite {
       s.run("import org.codehaus.plexus.util._").verifyResult("import org.codehaus.plexus.util._\n")
 
       // Check does SparkContext see classes defined by Scala interpreter.
-      s.run("case class Item(i: Int)").verifyResult("defined class Item\n")
+      // Scala 2.12's REPL reports `defined class Item`; Scala 2.13 emits the
+      // shorter `class Item`. Accept both.
+      s.run("case class Item(i: Int)").verifyResult(s"${optionalDefinedPrefixRegex}class Item\n")
+      // Scala 2.13's REPL prefixes with `val ` (e.g. `val rdd: ...`); 2.12
+      // omits it. Spark 4 (Scala 2.13) also prints a deprecation warning
+      // before the value binding because `parallelize` is now deprecated on
+      // SparkContext -- accept an optional warning header.
       s.run("val rdd = sc.parallelize(Array.fill(10){new Item(scala.util.Random.nextInt(1000))})")
-        .verifyResult("rdd.*")
+        .verifyResult(s"(?s)${optionalWarningPrefixRegex}${optionalValPrefixRegex}rdd.*")
       s.run("rdd.count()").verifyResult(".*= 10\n")
     }
   }
@@ -188,15 +213,17 @@ class InteractiveIT extends BaseIntegrationTestSuite {
   test("recover interactive session") {
     withNewSession(Spark) { s =>
       val stmt1 = s.run("1")
-      stmt1.verifyResult("res0: Int = 1\n")
+      // Scala 2.13's REPL renders value results as `val res0: Int = 1`
+      // whereas Scala 2.12 prints `res0: Int = 1`; accept both.
+      stmt1.verifyResult(s"${optionalValPrefixRegex}res0: Int = 1\n")
 
       restartLivy()
 
       // Verify session still exists.
       s.verifySessionIdle()
-      s.run("2").verifyResult("res1: Int = 2\n")
+      s.run("2").verifyResult(s"${optionalValPrefixRegex}res1: Int = 2\n")
       // Verify statement result is preserved.
-      stmt1.verifyResult("res0: Int = 1\n")
+      stmt1.verifyResult(s"${optionalValPrefixRegex}res0: Int = 1\n")
 
       s.stop()
 

pom.xml

@@ -81,16 +81,15 @@
     <asynchttpclient.version>2.10.1</asynchttpclient.version>
     <hadoop.scope>compile</hadoop.scope>
     <slf4j.version>1.7.36</slf4j.version>
-    <spark.scala-2.12.version>3.3.4</spark.scala-2.12.version>
-    <spark.version>${spark.scala-2.12.version}</spark.version>
+    <spark.version>4.1.2</spark.version>
     <kubernetes.client.version>5.6.0</kubernetes.client.version>
     <hive.version>3.0.0</hive.version>
     <commons-codec.version>1.15</commons-codec.version>
     <commons-lang3.version>3.17.0</commons-lang3.version>
     <httpclient.version>4.5.14</httpclient.version>
     <httpcore.version>4.4.16</httpcore.version>
-    <jackson.version>2.12.7</jackson.version>
-    <jackson-databind.version>2.12.7.1</jackson-databind.version>
+    <jackson.version>2.18.2</jackson.version>
+    <jackson-databind.version>2.18.2</jackson-databind.version>
     <jacoco.version>0.8.13</jacoco.version>
     <javax.servlet-api.version>3.1.0</javax.servlet-api.version>
     <jetty.version>9.4.56.v20240826</jetty.version>
@@ -103,7 +102,10 @@
          HealthCheckFilter at runtime (the class moved in Dropwizard 4.x). -->
     <metrics.version>4.2.19</metrics.version>
     <mockito.version>1.10.19</mockito.version>
-    <netty.version>4.1.86.Final</netty.version>
+    <!-- Spark 4.1.2 depends on Netty 4.2.7 and calls the new 4.2-only
+         `KQueueIoHandler`; pinning to a 4.1.x line here would let Livy's
+         `netty-all` win and hide that class from Spark's classpath. -->
+    <netty.version>4.2.7.Final</netty.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <!-- Scalatest 3.2.9 is required by Scalatra 2.8.x (needed for Scala
          2.13 support). All test suites are written against the 3.2 API,
@@ -113,7 +115,6 @@
          scalatest (e.g. 3.2.9.0 for scalatest 3.2.9). -->
     <scalatestplus.mockito.version>3.2.9.0</scalatestplus.mockito.version>
     <scalatra.version>2.8.4</scalatra.version>
-    <java.version>1.8</java.version>
     <owasp.version>12.1.9</owasp.version>
     <extraJavaTestArgs>
       -XX:+IgnoreUnrecognizedVMOptions
@@ -133,15 +134,21 @@
     <test.redirectToFile>true</test.redirectToFile>
     <execution.root>${user.dir}</execution.root>
     <spark.home>${execution.root}/dev/spark</spark.home>
-    <!-- profile controlled parameters, we are setting the defaults here so we can run "mvn package" with default profile values -->
-    <!-- hadoop2 -->
-    <hadoop.major-minor.version>2</hadoop.major-minor.version>
-    <hadoop.version>2.7.3</hadoop.version>
-    <scala.binary.version>2.12</scala.binary.version>
-    <scala.version>2.12.18</scala.version>
-    <java.version>1.8</java.version>
-    <py4j.version>0.10.9</py4j.version>
-    <json4s.version>3.5.3</json4s.version>
+    <!-- Profile-controlled parameters: defaults target Spark 4 + Scala 2.13 so
+         `mvn package` works without extra flags. Use -Pspark3 -Pscala-2.12 for
+         Spark 3 builds. -->
+    <hadoop.major-minor.version>3</hadoop.major-minor.version>
+    <hadoop.version>3.4.1</hadoop.version>
+    <scala.binary.version>2.13</scala.binary.version>
+    <!-- Match the Scala patch Spark 4.1.2 was built against. Spark's
+         Catalyst calls `MurmurHash3.caseClassHash$default$2()` which was
+         added in 2.13.17; older 2.13.x runs fail with NoSuchMethodError. -->
+    <scala.version>2.13.17</scala.version>
+    <java.version>17</java.version>
+    <!-- Minimum JDK for maven-enforcer; follows java.version (1.8 under -Pspark3). -->
+    <enforcer.java.version>${java.version}</enforcer.java.version>
+    <py4j.version>0.10.9.9</py4j.version>
+    <json4s.version>4.0.7</json4s.version>
     <spark.bin.name>spark-${spark.version}-bin-hadoop${hadoop.major-minor.version}</spark.bin.name>
     <spark.bin.download.url>
       https://archive.apache.org/dist/spark/spark-${spark.version}/${spark.bin.name}.tgz
@@ -1071,7 +1078,11 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-shade-plugin</artifactId>
-          <version>3.5.0</version>
+          <!-- 3.6.2 bundles an ASM version that understands Java 22 bytecode
+               (major-version 66). Jackson 2.18.2 ships JDK 22 multi-release
+               classes under META-INF/versions/22, which 3.5.0's older ASM
+               refused to shade with `Unsupported class file major version`. -->
+          <version>3.6.2</version>
         </plugin>
 
         <plugin>
@@ -1213,7 +1224,7 @@
         <configuration>
           <rules>
             <requireJavaVersion>
-              <version>${java.version}</version>
+              <version>${enforcer.java.version}</version>
             </requireJavaVersion>
             <requireMavenVersion>
               <version>[3.0.0,)</version>
@@ -1425,6 +1436,7 @@
       </properties>
     </profile>
     <profile>
+      <!-- Use with `-Pspark3` for Spark 3 builds. -->
       <id>scala-2.12</id>
       <properties>
         <scala.binary.version>2.12</scala.binary.version>
@@ -1442,6 +1454,7 @@
     </profile>
 
     <profile>
+      <!-- Use with `-Pscala-2.12` for Spark 3 builds. -->
       <id>spark3</id>
       <properties>
         <spark.version>3.5.6</spark.version>

repl/pom.xml

@@ -60,6 +60,13 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>livy-test-lib</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>
@@ -205,6 +212,15 @@
                   <include>org.json4s:json4s-ast_${scala.binary.version}</include>
                   <include>org.json4s:json4s-core_${scala.binary.version}</include>
                   <include>org.json4s:json4s-jackson_${scala.binary.version}</include>
+                  <!-- json4s 4.x moved `JsonMethods` into a separate
+                       `json4s-jackson-core` module; without shading it too,
+                       the shaded jar is missing `JsonMethods.class` and
+                       PythonInterpreter fails at runtime with
+                       NoClassDefFoundError. The include is scoped to the
+                       artifactId prefix so builds against older json4s
+                       versions (which don't publish this module) still
+                       succeed. -->
+                  <include>org.json4s:json4s-jackson-core_${scala.binary.version}</include>
                   <include>org.json4s:json4s-scalap_${scala.binary.version}</include>
                   <include>com.esotericsoftware:kryo-shaded</include>
                 </includes>