Merge pull request #14 from hotosm/feat/integrate-fmtm

Refactor splitter.py, add helper functions, ready for FMTM integration

Dockerfile

@@ -119,8 +119,8 @@ FROM runtime as ci
 ARG PYTHON_IMG_TAG
 COPY --from=extract-deps \
     /opt/python/requirements-ci.txt /opt/python/
-RUN mv /root/.local/bin/* /usr/local/bin/ \
-    && mv /root/.local/lib/python${PYTHON_IMG_TAG}/site-packages/* \
+RUN cp -r /root/.local/bin/* /usr/local/bin/ \
+    && cp -r /root/.local/lib/python${PYTHON_IMG_TAG}/site-packages/* \
     /usr/local/lib/python${PYTHON_IMG_TAG}/site-packages/ \
     && set -ex \
     && apt-get update \

README.md

@@ -45,7 +45,7 @@
 <!-- markdownlint-enable -->
 
 This is a program to split polygons into tasks using a variety of
-algorythms. It is a class that can be used by other projects, but also
+algorithms. It is a class that can be used by other projects, but also
 a standalone program. It was originally developed for the
 [FMTM](https://github.com/hotosm/fmtm/wiki) project, but then
 converted so it can be used by multiple projects.

docker-compose.yml

@@ -0,0 +1,60 @@
+# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# This file is part of fmtm-splitter.
+#
+#     fmtm-splitter is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     fmtm-splitter is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with fmtm-splitter.  If not, see <https:#www.gnu.org/licenses/>.
+#
+
+version: "3"
+
+networks:
+  net:
+    name: fmtm-splitter
+
+services:
+  splitter:
+    image: "ghcr.io/hotosm/fmtm-splitter:${TAG_OVERRIDE:-ci}"
+    build:
+      target: ci
+    container_name: fmtm-splitter
+    volumes:
+      # Mount local package
+      - ./fmtm_splitter:/root/.local/lib/python3.10/site-packages/fmtm_splitter
+      # Mount local tests
+      - ./tests:/data/tests
+    depends_on:
+      db:
+        condition: service_healthy
+    networks:
+      - net
+    restart: "unless-stopped"
+    command: "pytest"
+
+  db:
+    image: "postgis/postgis:${POSTGIS_TAG:-14-3.4-alpine}"
+    container_name: fmtm-splitter-db
+    environment:
+      - POSTGRES_USER=fmtm
+      - POSTGRES_PASSWORD=dummycipassword
+      - POSTGRES_DB=splitter
+    ports:
+      - "5439:5432"
+    networks:
+      - net
+    restart: "unless-stopped"
+    healthcheck:
+      test: pg_isready -U ${FMTM_DB_USER:-fmtm} -d ${FMTM_DB_NAME:-fmtm}
+      start_period: 5s
+      interval: 10s
+      timeout: 5s
+      retries: 3

docs/usage.md

@@ -1,8 +1,9 @@
 # fmtm-splitter
 
 This program splits a Polygon AOI into tasks using a varity of
-algorythms.
+algorithms.
 
+```bash
     options:
       -h, --help            show this help message and exit
       -v, --verbose         verbose output
@@ -17,22 +18,80 @@ algorythms.
       -s SOURCE, --source SOURCE
           Source data, Geojson or PG:[dbname]
       -c CUSTOM, --custom CUSTOM
-          Custom SQL query for database]
+          Custom SQL query for database
+      -db DATABASE, --dburl DATABASE
+          The database url string to custom sql
+```
 
 The data source for existing data can be either the data extract used
 by the XLSForm, or a postgresql database.
 
-# Examples
+## Examples
 
-    fmtm-splitter -b AOI
-    fmtm-splitter -v -b AOI -s data.geojson
-    fmtm-splitter -v -b AOI -s PG:colorado
+### Via Command Line
 
-    Where AOI is the boundary of the project as a polygon
-    And OUTFILE is a MultiPolygon output file,which defaults to fmtm.geojson
-    The task splitting defaults to squares, 50 meters across. If -m is used
-    then that also defaults to square splitting.
+```bash
+fmtm-splitter -b AOI
+fmtm-splitter -v -b AOI -s data.geojson
+fmtm-splitter -v -b AOI -s PG:colorado
+```
 
-    fmtm-splitter -b AOI -b 20 -c custom.sql
-    This will use a custom SQL query for splitting by map feature, and adjust task
-    sizes based on the number of buildings.
+> Where AOI is the boundary of the project as a polygon
+> And OUTFILE is a MultiPolygon output file,which defaults to fmtm.geojson
+> The task splitting defaults to squares, 50 meters across. If -m is used
+> then that also defaults to square splitting.
+
+### With Custom Query
+
+```bash
+fmtm-splitter -b AOI -b 20 -c custom.sql
+```
+
+> This will use a custom SQL query for splitting by map feature, and adjust task
+> sizes based on the number of buildings.
+
+### Via API
+
+#### Split By Square
+
+```python
+from fmtm_splitter.splitter import split_by_square
+
+features = split_by_square(
+    "path/to/your/file.geojson",
+    meters=100,
+)
+```
+
+#### Split By Features
+
+```python
+import geojson
+from fmtm_splitter.splitter import split_by_features
+
+aoi_json = geojson.load("/path/to/file.geojson")
+# Dump string to show that passing string json is possible too
+split_geom_json = geojson.dumps(geojson.load("/path/to/file.geojson"))
+
+features = split_by_features(
+    aoi_json,
+    split_geom_json,
+)
+```
+
+#### Split By SQL
+
+```python
+import geojson
+from fmtm_splitter.splitter import split_by_sql
+
+aoi_json = geojson.load("/path/to/file.geojson")
+extract_json = geojson.load("/path/to/file.geojson")
+
+features = split_by_sql(
+    aoi_json,
+    "postgresql://myuser:mypass@myhost:5432/mydb",
+    num_buildings=10,
+    osm_extract=extract_json,
+)
+```

fmtm_splitter/db.py

@@ -0,0 +1,97 @@
+# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# This file is part of fmtm-splitter.
+#
+#     fmtm-splitter is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     fmtm-splitter is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with fmtm-splitter.  If not, see <https:#www.gnu.org/licenses/>.
+#
+"""DB models for temporary tables in splitBySQL."""
+import logging
+from typing import Union
+from uuid import uuid4
+
+from geoalchemy2 import Geometry
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+)
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+from sqlalchemy.engine import Engine, create_engine
+from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
+
+log = logging.getLogger(__name__)
+
+
+def get_engine(db: Union[str, Session]):
+    """Get engine from existing Session, or connection string.
+
+    If `db` is a connection string, a new engine is generated.
+    """
+    if isinstance(db, Session):
+        return db.get_bind()
+    elif isinstance(db, str):
+        return create_engine(db)
+    else:
+        msg = "The `db` variable is not a valid string or Session"
+        log.error(msg)
+        raise ValueError(msg)
+
+
+def new_session(engine: Engine):
+    """Get session using engine.
+
+    Be sure to use in a with statement.
+    with new_session(conn) as session:
+        session.add(xxx)
+        session.commit()
+    """
+    return sessionmaker(engine)
+
+
+class Base(DeclarativeBase):
+    """Wrapper for DeclarativeBase creating all tables."""
+
+    pass
+
+
+class DbProjectAOI(Base):
+    """The AOI geometry for a project."""
+
+    __tablename__ = "project_aoi"
+
+    id = Column(UUID(as_uuid=True), primary_key=True, default=uuid4)
+    geom = Column(Geometry(geometry_type="GEOMETRY", srid=4326))
+    tags = Column(JSONB)
+
+
+class DbBuildings(Base):
+    """Associated OSM buildings for a project."""
+
+    __tablename__ = "ways_poly"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(String)
+    osm_id = Column(String)
+    geom = Column(Geometry(geometry_type="GEOMETRY", srid=4326))
+    tags = Column(JSONB)
+
+
+class DbOsmLines(Base):
+    """Associated OSM ways for a project."""
+
+    __tablename__ = "ways_line"
+
+    id = Column(Integer, primary_key=True)
+    project_id = Column(String)
+    geom = Column(Geometry(geometry_type="GEOMETRY", srid=4326))
+    tags = Column(JSONB)

fmtm_splitter/fmtm_algorithm.sql

@@ -38,7 +38,7 @@ WITH aoi AS (
 -- Combine the boundary of the AOI with the splitlines
 -- First extract the Area of Interest boundary as a line
 ,boundary AS (
-    SELECT ST_Boundary(geometry) AS geom
+    SELECT ST_Boundary(geom) AS geom
     FROM aoi
 )
 -- Then combine it with the splitlines
@@ -194,8 +194,7 @@ WITH splitpolygonswithcontents AS (
 )
 ,clusteredbuildingsnocombineduid AS (
 SELECT *,
-    -- ST_ClusterKMeans(geom, cast((b.numfeatures / :num_buildings) + 1 as integer))
-    ST_ClusterKMeans(geom, cast((b.numfeatures / {nbuildings}) + 1 as integer))
+    ST_ClusterKMeans(geom, cast((b.numfeatures / :num_buildings) + 1 as integer))
     over (partition by polyid) as cid
 FROM buildingstocluster b
 )