Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SEDONA-425] Add RS_Values and RS_Value to accept grid coordinates #1122

Merged
merged 23 commits into from
Nov 16, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
43a448e
Refactor notebooks to include SedonaKepler and Sedona 1.5.0
iGN5117 Oct 16, 2023
2f11a38
temp commit changing notebook and binder Pipfile
iGN5117 Oct 19, 2023
0e55f85
Merge branch 'master' of https://github.com/apache/sedona into sedona…
iGN5117 Nov 2, 2023
f5353ba
Merge branch 'master' of https://github.com/apache/sedona into sedona…
iGN5117 Nov 2, 2023
186d386
Added values() variant to accept Grid coordinates
prantogg Nov 14, 2023
f5a413c
Added java tests for values()
prantogg Nov 14, 2023
a7325cc
Added scala tests for RS_Values
prantogg Nov 14, 2023
c16c0ea
Added inferrable support for List<Double> and List<Geometry>
iGN5117 Nov 14, 2023
2410d94
Revert old changes
iGN5117 Nov 14, 2023
14e3f0d
Merge pull request #1 from iGN5117/develop_Nilesh_Inferrable_Change
prantogg Nov 14, 2023
49e4d84
Added grid coordinates variant of value()
prantogg Nov 15, 2023
bc70eb5
Added java tests for value with grid coordinates
prantogg Nov 15, 2023
09f9d96
Updated case class for RS_Value
prantogg Nov 15, 2023
7d58adf
Added scala tests for value with grid coordinates
prantogg Nov 15, 2023
3a03f80
Update docs for RS_Value and RS_Values
prantogg Nov 15, 2023
63cf004
Fix typo
prantogg Nov 15, 2023
67b2bc2
Update R Tests for RS_Values and RS_Value
prantogg Nov 15, 2023
a149465
Update value and values to accept optional band argument
prantogg Nov 15, 2023
9ab586f
Updated case classes for RS_Value and RS_Values
prantogg Nov 15, 2023
a629b6d
Updated Scala tests
prantogg Nov 15, 2023
4b075f9
Updated docs with non-band function signatures for RS_Value and RS_Va…
prantogg Nov 15, 2023
cc5b903
Revert "Update R Tests for RS_Values and RS_Value"
prantogg Nov 15, 2023
719e3f0
Added note to match CRS of raster and Point geometries
prantogg Nov 15, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.sedona.common.raster;

import org.apache.sedona.common.utils.RasterUtils;
import org.geotools.coverage.grid.GridCoordinates2D;
import org.geotools.coverage.grid.GridCoverage2D;
import org.geotools.geometry.DirectPosition2D;
import org.locationtech.jts.geom.*;
Expand All @@ -41,6 +42,18 @@ public static Double value(GridCoverage2D rasterGeom, Geometry geometry, int ban
return values(rasterGeom, Collections.singletonList(geometry), band).get(0);
}

public static Double value(GridCoverage2D rasterGeom, Geometry geometry) throws TransformException
{
return values(rasterGeom, Collections.singletonList(geometry), 1).get(0);
}

public static Double value(GridCoverage2D rasterGeom, int colX, int rowY, int band) throws TransformException
{
int[] xCoordinates = {colX};
int[] yCoordinates = {rowY};
return values(rasterGeom, xCoordinates, yCoordinates, band).get(0);
}

public static Geometry getPixelAsPolygon(GridCoverage2D raster, int colX, int rowY) throws TransformException, FactoryException {
int srid = RasterAccessors.srid(raster);
Point2D point2D1 = RasterUtils.getWorldCornerCoordinates(raster, colX, rowY);
Expand Down Expand Up @@ -77,12 +90,42 @@ public static Geometry getPixelAsPoint(GridCoverage2D raster, int colX, int rowY
}
return GEOMETRY_FACTORY.createPoint(pointCoord);
}
public static List<Double> values(GridCoverage2D rasterGeom, List<Geometry> geometries, int band) throws TransformException {

public static List<Double> values(GridCoverage2D rasterGeom, int[] xCoordinates, int[] yCoordinates, int band) throws TransformException {
RasterUtils.ensureBand(rasterGeom, band); // Check for invalid band index
int numBands = rasterGeom.getNumSampleDimensions();
if (band < 1 || band > numBands) {
// Invalid band index. Return nulls.
return geometries.stream().map(geom -> (Double) null).collect(Collectors.toList());

double noDataValue = RasterUtils.getNoDataValue(rasterGeom.getSampleDimension(band - 1));
List<Double> result = new ArrayList<>(xCoordinates.length);
double[] pixelBuffer = new double[numBands];

for (int i = 0; i < xCoordinates.length; i++) {
int x = xCoordinates[i];
int y = yCoordinates[i];

GridCoordinates2D gridCoord = new GridCoordinates2D(x, y);

try {
pixelBuffer = rasterGeom.evaluate(gridCoord, pixelBuffer);
double pixelValue = pixelBuffer[band - 1];
if (Double.compare(noDataValue, pixelValue) == 0) {
result.add(null);
} else {
result.add(pixelValue);
}
} catch (PointOutsideCoverageException e) {
// Points outside the extent should return null
result.add(null);
}
}

return result;
}

public static List<Double> values(GridCoverage2D rasterGeom, List<Geometry> geometries, int band) throws TransformException {
RasterUtils.ensureBand(rasterGeom, band); // Check for invalid band index
int numBands = rasterGeom.getNumSampleDimensions();

double noDataValue = RasterUtils.getNoDataValue(rasterGeom.getSampleDimension(band - 1));
double[] pixelBuffer = new double[numBands];

Expand Down Expand Up @@ -110,6 +153,10 @@ public static List<Double> values(GridCoverage2D rasterGeom, List<Geometry> geom
return result;
}

public static List<Double> values(GridCoverage2D rasterGeom, List<Geometry> geometries) throws TransformException {
return values(rasterGeom, geometries, 1);
}

private static Point ensurePoint(Geometry geometry) {
if (geometry instanceof Point) {
return (Point) geometry;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,6 @@ public void testSetSrid() throws FactoryException {
@Test
public void value() throws TransformException {
assertNull("Points outside of the envelope should return null.", PixelFunctions.value(oneBandRaster, point(1, 1), 1));
assertNull("Invalid band should return null.", PixelFunctions.value(oneBandRaster, point(378923, 4072346), 0));
assertNull("Invalid band should return null.", PixelFunctions.value(oneBandRaster, point(378923, 4072346), 2));

Double value = PixelFunctions.value(oneBandRaster, point(378923, 4072346), 1);
assertNotNull(value);
Expand All @@ -70,11 +68,30 @@ public void value() throws TransformException {
assertNull("Null should be returned for no data values.", PixelFunctions.value(oneBandRaster, point(378923, 4072376), 1));
}

@Test
public void valueWithGridCoords() throws TransformException {
int insideX = 1;
int insideY = 0;
int outsideX = 4;
int outsideY = 4;

Double insideValue = PixelFunctions.value(oneBandRaster, insideX, insideY, 1);
assertNotNull("Value should not be null for points inside the envelope.", insideValue);
assertNull("Points outside of the envelope should return null.", PixelFunctions.value(oneBandRaster, outsideX, outsideY, 1));

int noDataX = 0;
int noDataY = 0;

assertNull("Null should be returned for no data values.", PixelFunctions.value(oneBandRaster, noDataX, noDataY, 1));
}

@Test
public void valueWithMultibandRaster() throws TransformException {
// Multiband raster
assertEquals(9d, PixelFunctions.value(multiBandRaster, point(4.5d,4.5d), 3), 0.1d);
assertEquals(255d, PixelFunctions.value(multiBandRaster, point(4.5d,4.5d), 4), 0.1d);
assertEquals(4d, PixelFunctions.value(multiBandRaster, 2,2, 3), 0.1d);
assertEquals(255d, PixelFunctions.value(multiBandRaster, 3,4, 4), 0.1d);
}

@Test
Expand Down Expand Up @@ -175,17 +192,19 @@ public void values() throws TransformException {
assertEquals(2, values.size());
assertTrue(values.stream().allMatch(Objects::nonNull));

values = PixelFunctions.values(oneBandRaster, points, 0);
values = PixelFunctions.values(oneBandRaster, Arrays.asList(new Geometry[]{point(378923, 4072346), null}), 1);
assertEquals(2, values.size());
assertTrue("All values should be null for invalid band index.", values.stream().allMatch(Objects::isNull));
assertNull("Null geometries should return null values.", values.get(1));
}

values = PixelFunctions.values(oneBandRaster, points, 2);
assertEquals(2, values.size());
assertTrue("All values should be null for invalid band index.", values.stream().allMatch(Objects::isNull));
@Test
public void valuesWithGridCoords() throws TransformException {
int[] xCoordinates = {1, 0};
int[] yCoordinates = {0, 1};

values = PixelFunctions.values(oneBandRaster, Arrays.asList(new Geometry[]{point(378923, 4072346), null}), 1);
List<Double> values = PixelFunctions.values(oneBandRaster, xCoordinates, yCoordinates, 1);
assertEquals(2, values.size());
assertNull("Null geometries should return null values.", values.get(1));
assertTrue(values.stream().allMatch(Objects::nonNull));
}

private Point point(double x, double y) {
Expand Down
36 changes: 30 additions & 6 deletions docs/api/sql/Raster-operators.md
Original file line number Diff line number Diff line change
Expand Up @@ -1548,23 +1548,35 @@ Output:

### RS_Value

Introduction: Returns the value at the given point in the raster.
If no band number is specified it defaults to 1.
Introduction: Returns the value at the given point in the raster. If no band number is specified it defaults to 1.

Format:

`RS_Value (raster: Raster, point: Geometry)`

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please keep RS_Value (raster: Raster, point: Geometry) signature because we don't want to introduce any API breaking change.

Then revert the change R test because it is no longer needed.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have kept RS_Value (raster: Raster, point: Geometry) signature and reverted changes to R tests.

Note:

  • Only Point geometry variant of RS_Value and RS_Values can take band as optional parameter
  • Same currently not possible for the integer variant because of conflicting function signatures as shown below,
    RS_Value (raster: Raster, colX: Integer, colY: Integer)
    RS_Value (raster: Raster, point: Geometry, band: Integer)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you also add the explanation to the RS_Value and RS_Values doc: the input geometry point must be in the same CRS of the raster.

`RS_Value (raster: Raster, point: Geometry, band: Integer)`

`RS_Value (raster: Raster, colX: Integer, colY: Integer, band: Integer)`

Since: `v1.4.0`

Spark SQL Example:
!!!Note
The input geometry points must be in the same CRS as the raster. Ensure that all points' CRS matches the raster's CRS to get accurate values.

Spark SQL Examples:

- For Point Geometry:

```sql
SELECT RS_Value(raster, ST_Point(-13077301.685, 4002565.802)) FROM raster_table
```

- For Grid Coordinates:

```sql
SELECT RS_Value(raster, 3, 4, 1) FROM raster_table
```

Output:

```
Expand All @@ -1573,10 +1585,9 @@ Output:

### RS_Values

Introduction: Returns the values at the given points in the raster.
If no band number is specified it defaults to 1.
Introduction: Returns the values at the given points or grid coordinates in the raster. If no band number is specified it defaults to 1.

RS_Values is similar to RS_Value but operates on an array of points.
RS_Values is similar to RS_Value but operates on an array of points or grid coordinates.
RS_Values can be significantly faster since a raster only has to be loaded once for several points.

Format:
Expand All @@ -1585,15 +1596,28 @@ Format:

`RS_Values (raster: Raster, points: ARRAY[Geometry], band: Integer)`

`RS_Values (raster: Raster, xCoordinates: ARRAY[Integer], yCoordinates: ARRAY[Integer], band: Integer)`

Since: `v1.4.0`

!!!Note
The input geometry points must be in the same CRS as the raster. Ensure that all points' CRS matches the raster's CRS to get accurate values.

Spark SQL Example:

- For Array of Point geometries:

```sql
SELECT RS_Values(raster, Array(ST_Point(-1307.5, 400.8), ST_Point(-1403.3, 399.1)))
FROM raster_table
```

- For Arrays of grid coordinates:

```sql
SELECT RS_Values(raster, Array(4, 5), Array(3, 2), 1) FROM raster_table
```

Output:

```
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import org.apache.spark.sql.sedona_sql.expressions.implicits._
import org.apache.spark.sql.sedona_sql.expressions.raster.implicits._
import org.geotools.coverage.grid.GridCoverage2D

import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable`
import scala.reflect.runtime.universe.TypeTag
import scala.reflect.runtime.universe.Type
import scala.reflect.runtime.universe.typeOf
Expand Down Expand Up @@ -114,6 +115,10 @@ object InferrableType {
new InferrableType[Array[java.lang.Long]] {}
implicit val doubleArrayInstance: InferrableType[Array[Double]] =
new InferrableType[Array[Double]] {}
implicit val javaDoubleListInstance: InferrableType[java.util.List[java.lang.Double]] =
new InferrableType[java.util.List[java.lang.Double]] {}
implicit val javaGeomListInstance: InferrableType[java.util.List[Geometry]] =
new InferrableType[java.util.List[Geometry]] {}
}

object InferredTypes {
Expand All @@ -135,6 +140,10 @@ object InferredTypes {
case null => null
case arrayData: ArrayData => arrayData.toIntArray()
}
} else if (t =:= typeOf[java.util.List[Geometry]]) {
expr => input => expr.toGeometryList(input)
} else if (t =:= typeOf[java.util.List[java.lang.Double]]) {
expr => input => expr.toDoubleList(input)
} else {
expr => input => expr.eval(input)
}
Expand Down Expand Up @@ -162,14 +171,23 @@ object InferredTypes {
} else {
null
}
} else if (t =:= typeOf[Array[java.lang.Long]] || t =:= typeOf[Array[Long]] || t =:= typeOf[Array[Double]]) {
} else if (t =:= typeOf[Array[java.lang.Long]] || t =:= typeOf[Array[Long]] ||
t =:= typeOf[Array[Double]]) {
output =>
if (output != null) {
ArrayData.toArrayData(output)
} else {
null
}
} else if (t =:= typeOf[Array[Geometry]]) {
}else if (t =:= typeOf[java.util.List[java.lang.Double]]) {
output =>
if (output != null) {
ArrayData.toArrayData(output.asInstanceOf[java.util.List[java.lang.Double]].map(elem => elem))
}else {
null
}
}
else if (t =:= typeOf[Array[Geometry]] || t =:= typeOf[java.util.List[Geometry]]) {
output =>
if (output != null) {
ArrayData.toArrayData(output.asInstanceOf[Array[Geometry]].map(_.toGenericArrayData))
Expand All @@ -191,7 +209,7 @@ object InferredTypes {
def inferSparkType(t: Type): DataType = {
if (t =:= typeOf[Geometry]) {
GeometryUDT
} else if (t =:= typeOf[Array[Geometry]]) {
} else if (t =:= typeOf[Array[Geometry]] || t =:= typeOf[java.util.List[Geometry]]) {
DataTypes.createArrayType(GeometryUDT)
} else if (t =:= typeOf[GridCoverage2D]) {
RasterUDT
Expand All @@ -215,7 +233,7 @@ object InferredTypes {
DataTypes.createArrayType(IntegerType)
} else if (t =:= typeOf[Array[Long]] || t =:= typeOf[Array[java.lang.Long]]) {
DataTypes.createArrayType(LongType)
} else if (t =:= typeOf[Array[Double]]) {
} else if (t =:= typeOf[Array[Double]] || t =:= typeOf[java.util.List[java.lang.Double]]) {
DataTypes.createArrayType(DoubleType)
} else if (t =:= typeOf[Option[Boolean]]) {
BooleanType
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import org.apache.spark.sql.types.{ByteType, DataTypes}
import org.apache.spark.unsafe.types.UTF8String
import org.locationtech.jts.geom.{Geometry, GeometryFactory, Point}

import java.util

object implicits {

implicit class InputExpressionEnhancer(inputExpression: Expression) {
Expand Down Expand Up @@ -59,6 +61,42 @@ object implicits {
}
}

def toDoubleList(input: InternalRow): java.util.List[java.lang.Double] = {
inputExpression match {
case aware: SerdeAware =>
aware.evalWithoutSerialization(input).asInstanceOf[java.util.List[java.lang.Double]]
case _ =>
inputExpression.eval(input).asInstanceOf[ArrayData] match {
case arrayData: ArrayData =>
val length = arrayData.numElements()
val doubleList = new java.util.ArrayList[java.lang.Double]()
for (i <- 0 until length) {
doubleList.add(arrayData.getDouble(i))
}
doubleList.asInstanceOf[java.util.List[java.lang.Double]]
case _ => null
}
}
}

def toGeometryList(input: InternalRow): java.util.List[Geometry] = {
inputExpression match {
case aware: SerdeAware =>
aware.evalWithoutSerialization(input).asInstanceOf[java.util.List[Geometry]]
case _ =>
inputExpression.eval(input).asInstanceOf[ArrayData] match {
case arrayData: ArrayData =>
val length = arrayData.numElements()
val geometries = new java.util.ArrayList[Geometry]()
for (i <- 0 until length) {
geometries.add(arrayData.getBinary(i).toGeometry)
}
geometries.asInstanceOf[java.util.List[Geometry]]
case _ => null
}
}
}

def toInt(input: InternalRow): Int = {
inputExpression.eval(input).asInstanceOf[Int]
}
Expand Down
Loading
Loading