diff --git a/.github/workflows/core-ci.yml b/.github/workflows/core-ci.yml index 3ef8368..a080076 100644 --- a/.github/workflows/core-ci.yml +++ b/.github/workflows/core-ci.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - spark: ["3.0.1", "3.1.3", "3.2.4", "3.3.4"] + spark: ["3.0.3", "3.1.3", "3.2.4", "3.3.4", "3.4.3", "3.5.3"] runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 diff --git a/build.sbt b/build.sbt index 2bf0b03..0aed496 100644 --- a/build.sbt +++ b/build.sbt @@ -1,3 +1,5 @@ +import scala.language.postfixOps + Compile / scalafmtOnCompile := true organization := "com.github.mrpowers" @@ -5,12 +7,9 @@ name := "spark-daria" version := "1.2.3" -crossScalaVersions := Seq("2.12.15", "2.13.8") -scalaVersion := "2.12.15" - val versionRegex = """^(.*)\.(.*)\.(.*)$""".r -val scala2_13 = "2.13.14" +val scala2_13 = "2.13.15" val scala2_12 = "2.12.20" val sparkVersion = System.getProperty("spark.testVersion", "3.3.4") @@ -18,6 +17,7 @@ crossScalaVersions := { sparkVersion match { case versionRegex("3", m, _) if m.toInt >= 2 => Seq(scala2_12, scala2_13) case versionRegex("3", _, _) => Seq(scala2_12) + case versionRegex("4", _, _) => Seq(scala2_13) } } @@ -32,9 +32,9 @@ lazy val commonSettings = Seq( libraryDependencies ++= Seq( "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", "org.apache.spark" %% "spark-mllib" % sparkVersion % "provided", - "com.github.mrpowers" %% "spark-fast-tests" % "1.1.0" % "test", - "com.lihaoyi" %% "utest" % "0.7.11" % "test", - "com.lihaoyi" %% "os-lib" % "0.8.0" % "test" + "com.github.mrpowers" %% "spark-fast-tests" % "1.3.0" % "test", + "com.lihaoyi" %% "utest" % "0.8.2" % "test", + "com.lihaoyi" %% "os-lib" % "0.10.3" % "test" ), ) @@ -48,6 +48,18 @@ lazy val unsafe = (project in file("unsafe")) .settings( commonSettings, name := "unsafe", + Compile / unmanagedSourceDirectories ++= { + sparkVersion match { + case versionRegex(mayor, minor, _) => + (Compile / sourceDirectory).value ** s"*spark_*$mayor.$minor*" / "scala" get + } + }, + Test / unmanagedSourceDirectories ++= { + sparkVersion match { + case versionRegex(mayor, minor, _) => + (Compile / sourceDirectory).value ** s"*spark_*$mayor.$minor*" / "scala" get + } + }, ) testFrameworks += new TestFramework("com.github.mrpowers.spark.daria.CustomFramework")