Merge branch 'apache:master' into master

.github/workflows/docker-build.yml

@@ -25,13 +25,17 @@ jobs:
       fail-fast: true
       matrix:
         os: ['ubuntu-latest', 'macos-12']
+        spark: ['3.5.1', '3.4.3', '3.3.4']
         include:
-          - spark: 3.4.1
-            sedona: 1.4.1
-          - spark: 3.4.1
-            sedona: latest
-          - spark: 3.3.2
-            sedona: latest
+          - spark: 3.5.1
+            sedona: "latest"
+            geotools: "auto"
+          - spark: 3.4.3
+            sedona: 1.6.0
+            geotools: 28.2
+          - spark: 3.3.4
+            sedona: 1.6.0
+            geotools: 28.2
     runs-on: ${{ matrix.os }}
     defaults:
       run:
@@ -54,7 +58,12 @@ jobs:
       run: |
         brew install docker
         colima start
+        DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}
+        mkdir -p $DOCKER_CONFIG/cli-plugins
+        curl -SL https://github.com/docker/buildx/releases/download/v0.14.1/buildx-v0.14.1.darwin-amd64 -o $DOCKER_CONFIG/cli-plugins/docker-buildx
+        chmod +x $DOCKER_CONFIG/cli-plugins/docker-buildx
     - env:
         SPARK_VERSION: ${{ matrix.spark }}
         SEDONA_VERSION: ${{ matrix.sedona }}
-      run: ./docker/sedona-spark-jupyterlab/build.sh ${SPARK_VERSION} ${SEDONA_VERSION}
+        GEOTOOLS_VERSION: ${{ matrix.geotools }}
+      run: ./docker/sedona-spark-jupyterlab/build.sh ${SPARK_VERSION} ${SEDONA_VERSION} local ${GEOTOOLS_VERSION}

.github/workflows/example.yml

@@ -65,5 +65,5 @@ jobs:
         SPARK_VERSION: ${{ matrix.spark }}
         SPARK_COMPAT_VERSION: ${{ matrix.spark-compat }}
         SEDONA_VERSION: ${{ matrix.sedona }}
-      run: (cd examples/spark-sql;mvn clean install -Dspark.version=${SPARK_VERSION} -Dspark.compat.version=${SPARK_COMPAT_VERSION} -Dsedona.version=${SEDONA_VERSION};java -jar target/sedona-spark-example-1.0.0.jar)
-    - run: (cd examples/flink-sql;mvn clean install;java -jar target/sedona-flink-example-1.0.0.jar)
+      run: (cd examples/spark-sql;mvn clean install -Dspark.version=${SPARK_VERSION} -Dspark.compat.version=${SPARK_COMPAT_VERSION} -Dsedona.version=${SEDONA_VERSION};java -jar target/sedona-spark-example-1.6.0.jar)
+    - run: (cd examples/flink-sql;mvn clean install;java -jar target/sedona-flink-example-1.6.0.jar)

.gitignore

@@ -1,12 +1,12 @@
-/target/
+target
 /.idea
-/*.iml
+*.iml
 /*.ipr
 /*.iws
 /.settings/
 /.classpath
 /.project
-/dependency-reduced-pom.xml
+dependency-reduced-pom.xml
 /bin/
 /doc/
 /conf/

.pre-commit-config.yaml

@@ -19,12 +19,12 @@ repos:
         args: [--ignore-words=.github/linters/codespell.txt]
         exclude: ^docs/image|^spark/common/src/test/resources|^docs/usecases
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.2
+    rev: v0.4.4
     hooks:
       - id: ruff
         args: [--config=.github/linters/ruff.toml, --fix]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-ast
       - id: check-builtin-literals
@@ -53,13 +53,11 @@ repos:
       - id: forbid-submodules
       - id: mixed-line-ending
         exclude: \.csv$
-      - id: requirements-txt-fixer
-        files: ^docker/sedona-spark-jupyterlab/requirements\.txt$
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
         exclude: ^docs-overrides/main\.html$|\.Rd$
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.38.0
+    rev: v0.40.0
     hooks:
       - id: markdownlint
         name: Run markdownlint

binder/.dockerignore

@@ -0,0 +1,9 @@
+# Ignore everything
+*
+
+# Allow files and folders with a pattern starting with !
+!binder
+!docs/usecases
+!README.md
+!NOTICE
+!LICENSE

docs/usecases/Pipfile → binder/Pipfile

@@ -1,3 +1,5 @@
+# REMIND: Please run `pipenv lock` to update Pipfile.lock after editing this file
+
 [[source]]
 name = "pypi"
 url = "https://pypi.org/simple"
@@ -11,12 +13,12 @@ mkdocs="*"
 pytest-cov = "*"
 
 [packages]
-pandas="1.3.5"
-shapely="==1.8.4"
-geopandas="==0.11.1"
-pyspark="==3.4.0"
+pandas="==1.5.3"
+shapely="==2.0.4"
+geopandas="==0.14.4"
+pyspark="==3.4.2"
 attrs="*"
-apache-sedona="==1.5.1"
+apache-sedona="==1.6.0"
 matplotlib = "*"
 descartes = "*"
 keplergl = "==0.3.2"

docs/usecases/postBuild → binder/postBuild

@@ -1,12 +1,12 @@
 #Download Apache Spark
-wget https://archive.apache.org/dist/spark/spark-3.4.0/spark-3.4.0-bin-hadoop3.tgz
-tar -xzf spark-3.4.0-bin-hadoop3.tgz
+wget https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3.tgz
+tar -xzf spark-3.4.2-bin-hadoop3.tgz
 # Get AWS jars
 hadoop_s3_version=3.3.4
 aws_sdk_version=1.12.402
 
-curl https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${hadoop_s3_version}/hadoop-aws-${hadoop_s3_version}.jar -o $HOME/spark-3.4.0-bin-hadoop3/jars/hadoop-aws-${hadoop_s3_version}.jar
-curl https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${aws_sdk_version}/aws-java-sdk-bundle-${aws_sdk_version}.jar -o $HOME/spark-3.4.0-bin-hadoop3/jars/aws-java-sdk-bundle-${aws_sdk_version}.jar
+curl https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${hadoop_s3_version}/hadoop-aws-${hadoop_s3_version}.jar -o $HOME/spark-3.4.2-bin-hadoop3/jars/hadoop-aws-${hadoop_s3_version}.jar
+curl https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${aws_sdk_version}/aws-java-sdk-bundle-${aws_sdk_version}.jar -o $HOME/spark-3.4.2-bin-hadoop3/jars/aws-java-sdk-bundle-${aws_sdk_version}.jar
 
 #Tidy up
-rm spark-3.4.0-bin-hadoop3.tgz
+rm spark-3.4.2-bin-hadoop3.tgz

docs/usecases/start → binder/start

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-SPARK_HOME=$HOME/spark-3.4.0-bin-hadoop3
+export SPARK_HOME=$HOME/spark-3.4.2-bin-hadoop3
 export PATH=$SPARK_HOME/bin:$PATH
 export PYTHONPATH=$SPARK_HOME/python:$PYTHONPATH
 export PYSPARK_SUBMIT_ARGS="--master local[*] pyspark-shell"

common/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.apache.sedona</groupId>
         <artifactId>sedona-parent</artifactId>
-        <version>1.6.0</version>
+        <version>1.6.1-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>sedona-common</artifactId>

common/src/main/java/org/apache/sedona/common/Constructors.java

@@ -17,11 +17,7 @@
 import org.apache.sedona.common.enums.GeometryType;
 import org.apache.sedona.common.utils.FormatUtils;
 import org.apache.sedona.common.utils.GeoHashDecoder;
-import org.locationtech.jts.geom.Coordinate;
-import org.locationtech.jts.geom.CoordinateXYZM;
-import org.locationtech.jts.geom.Geometry;
-import org.locationtech.jts.geom.GeometryFactory;
-import org.locationtech.jts.geom.PrecisionModel;
+import org.locationtech.jts.geom.*;
 import org.locationtech.jts.io.ParseException;
 import org.locationtech.jts.io.WKBReader;
 import org.locationtech.jts.io.WKTReader;
@@ -70,6 +66,32 @@ public static Geometry geomFromWKB(byte[] wkb) throws ParseException {
         return new WKBReader().read(wkb);
     }
 
+    public  static Geometry pointFromWKB(byte[] wkb) throws ParseException {
+        return pointFromWKB(wkb, 0);
+    }
+
+    public static Geometry pointFromWKB(byte[] wkb, int srid) throws ParseException {
+        GeometryFactory geometryFactory = new GeometryFactory(new PrecisionModel(), srid);
+        Geometry geom =  new WKBReader(geometryFactory).read(wkb);
+        if (!(geom instanceof Point)) {
+            return null;
+        }
+        return geom;
+    }
+
+    public  static Geometry lineFromWKB(byte[] wkb) throws ParseException {
+        return lineFromWKB(wkb, 0);
+    }
+
+    public static Geometry lineFromWKB(byte[] wkb, int srid) throws ParseException {
+        GeometryFactory geometryFactory = new GeometryFactory(new PrecisionModel(), srid);
+        Geometry geom =  new WKBReader(geometryFactory).read(wkb);
+        if (!(geom instanceof LineString)) {
+            return null;
+        }
+        return geom;
+    }
+
     public static Geometry mLineFromText(String wkt, int srid) throws ParseException {
         if (wkt == null || !wkt.startsWith("MULTILINESTRING")) {
             return null;
@@ -136,6 +158,35 @@ public static Geometry pointZ(double x, double y, double z, int srid) {
         return geometryFactory.createPoint(new Coordinate(x, y, z));
     }
 
+    /**
+     * Creates a point from the given coordinate.
+     *
+     * @param x the x value
+     * @param y the y value
+     * @param m the m value
+     * @param srid Set to 0 if unknown
+     * @return The point geometry
+     */
+    public static Geometry pointM(double x, double y, double m, int srid) {
+        GeometryFactory geometryFactory = new GeometryFactory(new PrecisionModel(), srid);
+        return geometryFactory.createPoint(new CoordinateXYZM(x, y, 0, m));
+    }
+
+    /**
+     * Creates a point from the given coordinate.
+     *
+     * @param x the x value
+     * @param y the y value
+     * @param z the z value
+     * @param m the m value
+     * @param srid Set to 0 if unknown
+     * @return The point geometry
+     */
+    public static Geometry pointZM(double x, double y, double z, double m, int srid) {
+        GeometryFactory geometryFactory = new GeometryFactory(new PrecisionModel(), srid);
+        return geometryFactory.createPoint(new CoordinateXYZM(x, y, z, m));
+    }
+
     public static Geometry geomFromText(String geomString, String geomFormat, GeometryType geometryType) {
         FileDataSplitter fileDataSplitter = FileDataSplitter.getFileDataSplitter(geomFormat);
         FormatUtils<Geometry> formatMapper = new FormatUtils<>(fileDataSplitter, false, geometryType);

common/src/main/java/org/apache/sedona/common/Functions.java

@@ -47,6 +47,7 @@
 import org.locationtech.jts.operation.valid.TopologyValidationError;
 import org.locationtech.jts.precision.GeometryPrecisionReducer;
 import org.locationtech.jts.simplify.TopologyPreservingSimplifier;
+import org.locationtech.jts.triangulate.polygon.ConstrainedDelaunayTriangulator;
 import org.wololo.jts2geojson.GeoJSONWriter;
 
 import java.util.*;
@@ -390,6 +391,35 @@ public static Double z(Geometry geometry) {
         return null;
     }
 
+    public static Double m(Geometry geom) {
+        if (geom instanceof Point) {
+            return geom.getCoordinate().getM();
+        }
+        return null;
+    }
+
+    public static Double mMin(Geometry geometry) {
+        Coordinate[] points = geometry.getCoordinates();
+        double min = Double.MAX_VALUE;
+        for(int i=0; i < points.length; i++){
+            if(java.lang.Double.isNaN(points[i].getM()))
+                continue;
+            min = Math.min(points[i].getM(), min);
+        }
+        return min == Double.MAX_VALUE ? null : min;
+    }
+
+    public static Double mMax(Geometry geometry) {
+        Coordinate[] points = geometry.getCoordinates();
+        double max = - Double.MAX_VALUE;
+        for (int i=0; i < points.length; i++) {
+            if(java.lang.Double.isNaN(points[i].getM()))
+                continue;
+            max = Math.max(points[i].getM(), max);
+        }
+        return max == -Double.MAX_VALUE ? null : max;
+    }
+
     public static double xMin(Geometry geometry) {
         Coordinate[] points = geometry.getCoordinates();
         double min = Double.MAX_VALUE;
@@ -448,6 +478,11 @@ public static Double zMin(Geometry geometry) {
         return min == Double.MAX_VALUE ? null : min;
     }
 
+    public static boolean hasM(Geometry geom) {
+        Coordinate coord = geom.getCoordinate();
+        return !Double.isNaN(coord.getM());
+    }
+
     public static Geometry flipCoordinates(Geometry geometry) {
         GeomUtils.flipCoordinates(geometry);
         return geometry;
@@ -893,6 +928,10 @@ private static boolean checkIfPolygonCW(Polygon geom) {
         return isExteriorRingCW && isInteriorRingCW;
     }
 
+    public static Geometry triangulatePolygon(Geometry geom) {
+        return ConstrainedDelaunayTriangulator.triangulate(geom);
+    }
+
     public static double lineLocatePoint(Geometry geom, Geometry point)
     {
         double length = geom.getLength();

common/src/test/java/org/apache/sedona/common/ConstructorsTest.java

@@ -143,4 +143,22 @@ public void pointZ() {
         assertEquals(4326, point.getSRID());
         assertEquals("POINT Z(0 1 2)", Functions.asWKT(point));
     }
+
+    @Test
+    public void pointM() {
+        Geometry point = Constructors.pointM(0.0d, 1.0d, 2.0d, 4326);
+
+        assertTrue(point instanceof Point);
+        assertEquals(4326, point.getSRID());
+        assertEquals("SRID=4326;POINT ZM(0 1 0 2)", Functions.asEWKT(point));
+    }
+
+    @Test
+    public void pointZM() {
+        Geometry point = Constructors.pointZM(0.0d, 1.0d, 2.0d, 10.0, 4326);
+
+        assertTrue(point instanceof Point);
+        assertEquals(4326, point.getSRID());
+        assertEquals("POINT ZM(0 1 2 10)", Functions.asWKT(point));
+    }
 }