Merge pull request #182 from ecmwf/develop

Release 1.0.4

.gitignore

@@ -16,4 +16,11 @@ site
 example_eo
 example_mri
 .mypy_cache
-*.req
+*.req
+polytope_python.egg-info
+new_test_venv_polytope
+new_venv_polytope
+polytope_venv
+polytope_venv_latest
+new_updated_numpy_venv
+newest-polytope-venv

.readthedocs.yaml

@@ -0,0 +1,33 @@
+# Read the Docs configuration file for MkDocs projects
+
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+
+# Required
+
+version: 2
+
+
+# Set the version of Python and other tools you might need
+
+build:
+
+  os: ubuntu-22.04
+
+  tools:
+
+    python: "3.12"
+
+
+mkdocs:
+
+  configuration: mkdocs.yml
+
+
+# Optionally declare the Python requirements required to build your docs
+
+python:
+
+  install:
+
+  - requirements: docs/requirements.txt

docs/requirements.txt

@@ -1,3 +1,3 @@
-jinja2<3.1.0
+jinja2>=3.1.3
 Markdown<3.2
 mkdocs>=1.0

examples/3D_shipping_route.py

@@ -15,6 +15,9 @@ class Test:
     def setup_method(self):
         ds = data.from_source("file", "./examples/data/winds.grib")
         array = ds.to_xarray()
+        print(array.number)
+        print(array.surface)
+        print(array.time)
         array = array.isel(time=0).isel(surface=0).isel(number=0).u10
         self.array = array
         self.slicer = HullSlicer()

performance/fdb_performance_3D.py

@@ -20,6 +20,7 @@ def setup_method(self, method):
             "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}},
             "step": {"transformation": {"type_change": "int"}},
             "levelist": {"transformation": {"type_change": "int"}},
+            "longitude": {"transformation": {"cyclic": [0, 360]}},
         }
         self.config = {"class": "od", "expver": "0001", "levtype": "sfc"}
         self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options)

performance/fdb_slice_many_numbers_timeseries.py

@@ -1,23 +1,50 @@
 import time
 
 import pandas as pd
+import pygribjump as gj
 
-from polytope.datacube.backends.fdb import FDBDatacube
 from polytope.polytope import Polytope, Request
 from polytope.shapes import All, Point, Select
 
 time1 = time.time()
 # Create a dataarray with 3 labelled axes using different index types
+
+# config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"}
 options = {
-    "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}},
-    "date": {"merge": {"with": "time", "linkers": ["T", "00"]}},
-    "step": {"type_change": "int"},
-    "number": {"type_change": "int"},
-    "longitude": {"cyclic": [0, 360]},
+    "axis_config": [
+        {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]},
+        {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]},
+        {
+            "axis_name": "date",
+            "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}],
+        },
+        {
+            "axis_name": "values",
+            "transformations": [
+                {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}
+            ],
+        },
+        {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]},
+        {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]},
+    ],
+    "compressed_axes_config": [
+        "longitude",
+        "latitude",
+        "levtype",
+        "step",
+        "date",
+        "domain",
+        "expver",
+        "param",
+        "class",
+        "stream",
+        "type",
+        "number",
+    ],
+    "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"},
 }
-config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"}
-fdbdatacube = FDBDatacube(config, axis_options=options)
-self_API = Polytope(datacube=fdbdatacube, axis_options=options)
+fdbdatacube = gj.GribJump()
+self_API = Polytope(datacube=fdbdatacube, options=options)
 
 print(time.time() - time1)
 
@@ -39,7 +66,14 @@
     Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"),
     All("number"),
 )
+time3 = time.time()
 result = self_API.retrieve(request)
+time4 = time.time()
+print("Polytope time")
+print(self_API.time)
 print(time.time() - time1)
 print(time.time() - time2)
+print(time4 - time3)
 print(len(result.leaves))
+print([len(leaf.result) for leaf in result.leaves])
+result.pprint()

performance/performance_many_num_steps.py

@@ -0,0 +1,59 @@
+import time
+
+import numpy as np
+import pandas as pd
+from earthkit import data
+
+from polytope.datacube.backends.fdb import FDBDatacube
+from polytope.polytope import Polytope, Request
+from polytope.shapes import All, Point, Select, Span
+
+time1 = time.time()
+# Create a dataarray with 3 labelled axes using different index types
+options = {
+    "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}},
+    "longitude": {"cyclic": [0, 360]},
+    "latitude": {"reverse": {True}},
+}
+
+ds = data.from_source("file", "~/Downloads/ensemble-timeseries_v2.grib")
+array = ds.to_xarray().t2m
+print(array)
+
+
+config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"}
+fdbdatacube = FDBDatacube(config, axis_options=options)
+# self_API = Polytope(datacube=fdbdatacube, axis_options=options)
+self_API = Polytope(datacube=array, axis_options=options)
+
+print(time.time() - time1)
+
+total_polytope_time = 0
+for i in range(10):
+    time2 = time.time()
+
+request = Request(
+    Span("step", np.timedelta64(0, "s"), np.timedelta64(5 * 24 * 3600, "s")),
+    # Select("levtype", ["sfc"]),
+    # Select("date", [pd.Timestamp("20231205T000000")]),
+    Select("time", [pd.Timestamp("20231205T000000")]),
+    Select("surface", [0]),
+    # Select("domain", ["g"]),
+    # Select("expver", ["0001"]),
+    # Select("param", ["167"]),
+    # Select("class", ["od"]),
+    # Select("stream", ["enfo"]),
+    # Select("type", ["pf"]),
+    # Select("latitude", [0.035149384216], method="surrounding"),
+    Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"),
+    All("number"),
+)
+time3 = time.time()
+result = self_API.retrieve(request)
+time4 = time.time()
+print(time.time() - time1)
+print(time.time() - time2)
+print(time4 - time3)
+print(len(result.leaves))
+print([len(leaf.result) for leaf in result.leaves])
+result.pprint()