feat(core): df.geo polygon testing

[maartenbreddels]

Example usage coming

[maartenbreddels]

prepare dataframe

import vaex
import numpy as np
import plotly.express as px
df = vaex.open('s3://vaex/taxi/yellow_taxi_2009_2015_f32.hdf5?anon=true')
df = df[516794416:695338739+1]  # take 2012
df

prepare polygons from geojson

import json
import collections


geo_filename = '/Users/maartenbreddels/Downloads/taxi_zones.geojson'
with open(geo_filename) as f:
    geo_json = json.load(f)

features = geo_json['features']
borough_polygons = collections.defaultdict(list)
zone_polygons = collections.defaultdict(list)
list_of_polygons = []
for i, feature in enumerate(features[:]):
    properties = feature['properties']
    geo = feature['geometry']

    polygons = [np.array(polygon_set[0]).T for polygon_set in geo['coordinates']]
    borough_polygons[properties['borough']].extend(polygons)
    zone_polygons[properties['zone']].extend(polygons)

calculate polygon index

df['pickup_borough'] = df.geo.inside_which_polygons(df.pickup_longitude, df.pickup_latitude, borough_polygons.values())
df['pickup_zone'] = df.geo.inside_which_polygons(df.pickup_longitude, df.pickup_latitude, zone_polygons.values())
df.materialize(virtual_column='pickup_borough', inplace=True)
df.materialize(virtual_column='pickup_zone', inplace=True)

aggregate data

# 
dfg = df.groupby('pickup_zone', agg='count')
keys = list(zone_polygons.keys())
mapper = {i:keys[i] for i in range(len(keys))}
# astype is needed for some reason?
dfg['zone_name'] = dfg.pickup_zone.astype('int').map(mapper)
dfg['logcount'] = np.log10(dfg['count']).values

plot it with plotly

import plotly.express as px

fig = px.choropleth_mapbox(dfg.to_dict(virtual=True), geojson=geo_json, color="logcount",
                           locations="zone_name", featureidkey="properties.zone",
                           mapbox_style="carto-positron",
                           zoom=7, center = {"lat": 40.7, "lon": -73.99},
                           opacity=0.5,
                           
                          )
fig.update_layout(margin={"r":0,"t":0,"l":0,"b":0})
fig.show()

Json: https://toddwschneider.carto.com/viz/2961a180-ffb1-11e6-a29f-0e233c30368f/public_map
Other json dataset: https://gist.github.com/HenryQW/c2a353415a3ce88f829c3b1199ee3c17

[maartenbreddels]

I used https://mapshaper.org/ to reduce some geojson files, but that messed it up a bit, the following code should work:

import json
import collections


geo_filename = '/Users/maartenbreddels/Downloads/taxi_zones-tiny.json'
with open(geo_filename) as f:
    geo_json = json.load(f)

features = geo_json['features']
borough_polygons = collections.defaultdict(list)
zone_polygons = collections.defaultdict(list)
list_of_polygons = []
for i, feature in enumerate(features[:]):
    properties = feature['properties']
    geo = feature['geometry']

    polygons = []
    for polygon in geo['coordinates']:
        polygon = np.array(polygon)
        if polygon.ndim == 3:
            polygon = polygon[0]
        polygon = polygon.T
        assert polygon.shape[0] == 2
        assert polygon.ndim == 2
        polygons.append(polygon)

    borough_polygons[properties['borough']].extend(polygons)
    zone_polygons[properties['zone']].extend(polygons)

taxi_zones-tiny.json.zip

[maartenbreddels]

The AppVeyor failure is worrisome, but not related.