diff --git a/build.sbt b/build.sbt index 2ecd69b95..2c862b2d3 100644 --- a/build.sbt +++ b/build.sbt @@ -3,6 +3,7 @@ import sbt.{Compile, Def} val stableVersion = "0.0.3" val sparkDefaultShortVersion = "3.2" +val spark24Version = "2.4.8" val spark30Version = "3.0.3" val spark31Version = "3.1.3" val spark32Version = "3.2.1" @@ -10,10 +11,12 @@ val spark32Version = "3.2.1" val versionRegex = """^(.*)\.(.*)\.(.*)$""".r val versionRegexShort = """^(.*)\.(.*)$""".r +val scala211 = "2.11.12" val scala212 = "2.12.15" val scala213 = "2.13.8" val parserSparkVersion: String => String = { + case versionRegexShort("2", "4") => spark24Version case versionRegexShort("3", "0") => spark30Version case versionRegexShort("3", "1") => spark31Version case versionRegexShort("3", "2") => spark32Version @@ -25,11 +28,17 @@ val sparkLong2ShortVersion: String => String = { case versionRegex(a, b, _) => } val scalaVersionSelect: String => String = { + case versionRegex("2", _, _) => scala211 case versionRegex("3", "0", _) => scala212 case versionRegex("3", "1", _) => scala212 case versionRegex("3", "2", _) => scala212 } +val catsVersion: String => String = { + case versionRegex("2", _, _) => "2.0.0" + case _ => "2.7.0" +} + ThisBuild / organization := "org.hablapps" ThisBuild / homepage := Some(url("https://github.com/hablapps/doric")) ThisBuild / licenses := List( @@ -98,8 +107,8 @@ lazy val core = project scalaVersion := scalaVersionSelect(sparkVersion.value), libraryDependencies ++= Seq( "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", // scala-steward:off - "org.typelevel" %% "cats-core" % "2.7.0", - "com.lihaoyi" %% "sourcecode" % "0.2.8", + "org.typelevel" %% "cats-core" % catsVersion(sparkVersion.value), + "com.lihaoyi" %% "sourcecode" % "0.2.8", "com.github.mrpowers" %% "spark-fast-tests" % "1.2.0" % "test", "org.scalatest" %% "scalatest" % "3.2.11" % "test" ), diff --git a/core/src/main/scala/doric/DoricColumn.scala b/core/src/main/scala/doric/DoricColumn.scala index 239b79326..af5f4e8f6 100644 --- a/core/src/main/scala/doric/DoricColumn.scala +++ b/core/src/main/scala/doric/DoricColumn.scala @@ -1,7 +1,7 @@ package doric import cats.data.{Kleisli, NonEmptyChain, Validated} -import cats.implicits.{catsSyntaxApplicativeId, catsSyntaxTuple2Semigroupal, catsSyntaxValidatedId, catsSyntaxValidatedIdBinCompat0} +import cats.implicits._ import doric.sem.{ColumnTypeError, DoricSingleError, Location, SparkErrorWrapper} import doric.syntax.ColGetters import doric.types.{LiteralSparkType, SparkType} diff --git a/core/src/main/scala/doric/DoricJoinColumn.scala b/core/src/main/scala/doric/DoricJoinColumn.scala index a2be77dac..4531a35f4 100644 --- a/core/src/main/scala/doric/DoricJoinColumn.scala +++ b/core/src/main/scala/doric/DoricJoinColumn.scala @@ -1,6 +1,6 @@ package doric -import cats.implicits.catsSyntaxTuple2Semigroupal +import cats.implicits._ import doric.syntax.ColGetters import org.apache.spark.sql.Column diff --git a/core/src/main/scala/doric/JoinSideDoricColumn.scala b/core/src/main/scala/doric/JoinSideDoricColumn.scala index e7c2c59f8..a484d13db 100644 --- a/core/src/main/scala/doric/JoinSideDoricColumn.scala +++ b/core/src/main/scala/doric/JoinSideDoricColumn.scala @@ -1,7 +1,7 @@ package doric import cats.data.Kleisli -import cats.implicits.catsSyntaxTuple2Semigroupal +import cats.implicits._ import org.apache.spark.sql.{Column, Dataset} diff --git a/core/src/main/scala/doric/sem/TransformOps.scala b/core/src/main/scala/doric/sem/TransformOps.scala index ac0a856c6..7990fd710 100644 --- a/core/src/main/scala/doric/sem/TransformOps.scala +++ b/core/src/main/scala/doric/sem/TransformOps.scala @@ -1,7 +1,7 @@ package doric package sem -import cats.implicits.toTraverseOps +import cats.implicits._ import org.apache.spark.sql.{DataFrame, Dataset} import org.apache.spark.sql.doric.DataFrameExtras diff --git a/core/src/main/scala/doric/syntax/AggregationColumns.scala b/core/src/main/scala/doric/syntax/AggregationColumns.scala index a817a642f..a194e23bd 100644 --- a/core/src/main/scala/doric/syntax/AggregationColumns.scala +++ b/core/src/main/scala/doric/syntax/AggregationColumns.scala @@ -1,7 +1,7 @@ package doric package syntax -import cats.implicits.{catsSyntaxTuple2Semigroupal, toTraverseOps} +import cats.implicits._ import doric.types.NumericType import org.apache.spark.sql.{Column, functions => f} diff --git a/core/src/main/scala/doric/syntax/BinaryColumns.scala b/core/src/main/scala/doric/syntax/BinaryColumns.scala index d5f87e1ab..f50842db2 100644 --- a/core/src/main/scala/doric/syntax/BinaryColumns.scala +++ b/core/src/main/scala/doric/syntax/BinaryColumns.scala @@ -1,7 +1,7 @@ package doric package syntax -import cats.implicits.toTraverseOps +import cats.implicits._ import doric.types.{BinaryType, SparkType} import org.apache.spark.sql.{functions => f} diff --git a/core/src/main/scala/doric/syntax/StringColumns.scala b/core/src/main/scala/doric/syntax/StringColumns.scala index 724dd7c62..02f1ccdc7 100644 --- a/core/src/main/scala/doric/syntax/StringColumns.scala +++ b/core/src/main/scala/doric/syntax/StringColumns.scala @@ -1,7 +1,7 @@ package doric package syntax -import cats.implicits.{catsSyntaxTuple2Semigroupal, catsSyntaxTuple3Semigroupal, catsSyntaxTuple4Semigroupal, toTraverseOps} +import cats.implicits._ import org.apache.spark.sql.{Column, functions => f} import org.apache.spark.sql.catalyst.expressions._ diff --git a/core/src/main/scala/doric/syntax/TimestampColumns.scala b/core/src/main/scala/doric/syntax/TimestampColumns.scala index 414e86b02..3052515e4 100644 --- a/core/src/main/scala/doric/syntax/TimestampColumns.scala +++ b/core/src/main/scala/doric/syntax/TimestampColumns.scala @@ -1,14 +1,12 @@ package doric package syntax -import cats.implicits.catsSyntaxTuple2Semigroupal +import cats.implicits._ import doric.types.TimestampType -import org.apache.spark.sql.catalyst.expressions.{FromUTCTimestamp, ToUTCTimestamp} -import org.apache.spark.sql.{Column, functions => f} - import java.sql.Timestamp -import org.apache.spark.sql.{functions => f} +import org.apache.spark.sql.{Column, functions => f} +import org.apache.spark.sql.catalyst.expressions.{FromUTCTimestamp, ToUTCTimestamp} private[syntax] trait TimestampColumns { diff --git a/core/src/main/scala/doric/syntax/TypeMatcher.scala b/core/src/main/scala/doric/syntax/TypeMatcher.scala index d045e2047..e31916c1f 100644 --- a/core/src/main/scala/doric/syntax/TypeMatcher.scala +++ b/core/src/main/scala/doric/syntax/TypeMatcher.scala @@ -2,7 +2,7 @@ package doric package syntax import cats.data.Kleisli -import cats.implicits.catsSyntaxValidatedIdBinCompat0 +import cats.implicits._ import doric.sem.{ColumnMultyTypeError, Location} import doric.types.SparkType diff --git a/core/src/main/scala/doric/syntax/WhenBuilder.scala b/core/src/main/scala/doric/syntax/WhenBuilder.scala index 3a49dad69..6ef45d8dc 100644 --- a/core/src/main/scala/doric/syntax/WhenBuilder.scala +++ b/core/src/main/scala/doric/syntax/WhenBuilder.scala @@ -1,9 +1,10 @@ package doric package syntax -import cats.implicits.{catsSyntaxApplicativeId, catsSyntaxTuple2Semigroupal, catsSyntaxTuple3Semigroupal} +import cats.implicits._ import doric.types.{Casting, SparkType} -import org.apache.spark.sql.functions.{lit => sparkLit, when => sparkWhen} + +import org.apache.spark.sql.functions.{when => sparkWhen} import org.apache.spark.sql.Column final private[doric] case class WhenBuilder[T]( @@ -13,6 +14,7 @@ final private[doric] case class WhenBuilder[T]( /** * Marks the rest of cases as null values of the provided type + * * @param dt * Type class for spark data type * @return diff --git a/core/src/main/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala b/core/src/main/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala index 4d4ec1e7b..0b9263670 100644 --- a/core/src/main/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala +++ b/core/src/main/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala @@ -1,6 +1,6 @@ package org.apache.spark.sql.doric -import cats.implicits.toTraverseOps +import cats.implicits._ import doric.{DoricColumn, DoricValidated} import org.apache.spark.sql.{DataFrame, RelationalGroupedDataset} diff --git a/core/src/main/spark_3.0_3.1/scala/doric/syntax/BinaryColumns30_31.scala b/core/src/main/spark_3.0_3.1/scala/doric/syntax/BinaryColumns30_31.scala index 453cb102e..31c996533 100644 --- a/core/src/main/spark_3.0_3.1/scala/doric/syntax/BinaryColumns30_31.scala +++ b/core/src/main/spark_3.0_3.1/scala/doric/syntax/BinaryColumns30_31.scala @@ -1,7 +1,7 @@ package doric package syntax -import cats.implicits.catsSyntaxTuple2Semigroupal +import cats.implicits._ import doric.types.{BinaryType, SparkType} import org.apache.spark.sql.catalyst.expressions.Decode diff --git a/core/src/main/spark_3.2/scala/doric/syntax/BinaryColumns32.scala b/core/src/main/spark_3.2/scala/doric/syntax/BinaryColumns32.scala index 1c0a35c72..4b3231a8e 100644 --- a/core/src/main/spark_3.2/scala/doric/syntax/BinaryColumns32.scala +++ b/core/src/main/spark_3.2/scala/doric/syntax/BinaryColumns32.scala @@ -1,7 +1,7 @@ package doric package syntax -import cats.implicits.catsSyntaxTuple2Semigroupal +import cats.implicits._ import doric.types.{BinaryType, SparkType} import org.apache.spark.sql.Column