Merge pull request #12 from ecmwf/develop

API and config changes

README.md

@@ -54,9 +54,10 @@ pip install -e .
 
 ## Example
 
-**Create time series request**: Create a request for a time series request using the time series feature, set options and config for use by polytope feature extraction. NB: Assumes data is in a local FDB.
+**Create time series request**: Create a request for a time series request using the time series feature, set options for use by polytope feature extraction. "grib" indicates the type of data in this case. NB: Assumes data is in a local FDB.
 
 ```python
+from polytope_mars.api import PolytopeMars
 
 request = {
     "class": "od",
@@ -69,12 +70,31 @@ request = {
     "domain" : "g",
     "param" : "228/49/164/165/166/167",
     "number" : "1/to/5",
+    "step" : "0/1"
     "feature" : {
         "type" : "timeseries",
-        "points": [[51.5073219, 2.1]],
-        "start": 0,
-        "end" : 9
+        "points" : [[51.5073219, 2.1]],
+        "axis" : "step",
     },
 }
 
+result = PolytopeMars().extract(request)
+
 ```
+
+If the user provides no arguments to PolytopeMars then a config is loaded from the default locations:
+
+1. System-wide configuration in /etc/polytope_mars/config.json (and yaml)
+2. User configuration in ~/.polytope_mars.json (and yaml)
+
+The user can also pass in a config as a python dictionary to PolytopeMars for a custom config at runtime. 
+
+Result will be a coverageJSON file with the requested data if it is available, further manipulation of the coverage can be made using [covjsonkit](https://github.com/ecmwf/covjsonkit).
+
+### Config
+
+An example config can be found here [example_config.json](example_config.json). This can be edited to change any of the fields. The config is made up of three main components. 
+
+1. **datacube:** This option is used to set up what type of datacube is being used at the moment, currently only gribjump is supported.
+2. **options** These are the options used by polytope for interpreting the data available.
+3. **coverageconfig** These options are used by convjsonkit to parse the output of polytope into coverageJSON.

example_config.json

@@ -0,0 +1 @@
+{"datacube": {"type": "gribjump", "config": "config.yaml", "uri": "http://localhost:8001"}, "options": {"axis_config": [{"axis_name": "date", "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}]}, {"axis_name": "values", "transformations": [{"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"], "local": null}]}, {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": true}]}, {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0.0, 360.0]}]}, {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}], "compressed_axes_config": ["longitude", "latitude", "levtype", "step", "date", "domain", "expver", "param", "class", "stream", "type"], "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"}, "alternative_axes": []}, "coverageconfig": {"param_db": "ecmwf"}}

polytope_mars/api.py

@@ -1,12 +1,15 @@
 import json
 from typing import List
 
+import pandas as pd
 import pygribjump as gj
+from conflator import Conflator
 from covjsonkit.api import Covjsonkit
 from polytope import shapes
 from polytope.engine.hullslicer import HullSlicer
 from polytope.polytope import Polytope, Request
 
+from .config import PolytopeMarsConfig
 from .features.boundingbox import BoundingBox
 from .features.frame import Frame
 from .features.path import Path
@@ -27,16 +30,17 @@
 
 
 class PolytopeMars:
-    def __init__(self, datacube_type, datacube_options):
-        # Initialise polytope
-        # fdbdatacube = FDBDatacube(
-        #    datacube_config, axis_options=datacube_options
-        # )  # noqa: E501
-        # slicer = HullSlicer()
-        # self.api = Polytope(datacube=fdbdatacube, engine=slicer)
-
-        self.datacube_type = datacube_type
-        self.datacube_options = datacube_options
+    def __init__(self, config=None):
+        # Initialise polytope-mars configuration
+
+        # If no config check default locations
+        if config is None:
+            self.conf = Conflator(
+                app_name="polytope_mars", model=PolytopeMarsConfig
+            ).load()
+        # else initialise with provided config
+        else:
+            self.conf = PolytopeMarsConfig.model_validate(config)
 
         self.coverage = {}
 
@@ -62,6 +66,11 @@ def extract(self, request):
         except KeyError:
             raise KeyError("The 'feature' does not contain a 'type' keyword")
 
+        if feature_type == "timeseries":
+            timeseries_type = feature_config["axis"]
+        else:
+            timeseries_type = None
+
         feature = self._feature_factory(feature_type, feature_config)
 
         feature.validate(request)
@@ -72,29 +81,27 @@ def extract(self, request):
 
         preq = Request(*shapes)
 
-        # TODO: make polytope request to get data
-
-        if self.datacube_type == "grib":
+        if self.conf.datacube.type == "gribjump":
             fdbdatacube = gj.GribJump()
         else:
             raise NotImplementedError(
-                f"Datacube type '{self.datacube_type}' not found"
+                f"Datacube type '{self.conf.datacube.type}' not found"
             )  # noqa: E501
         slicer = HullSlicer()
         self.api = Polytope(
-            # request=preq,
             datacube=fdbdatacube,
             engine=slicer,
-            options=self.datacube_options,
+            options=self.conf.options.model_dump(),
         )
-        # result = API.retrieve(request)
-
         result = self.api.retrieve(preq)
-        # result.pprint()
+        encoder = Covjsonkit(self.conf.coverageconfig.model_dump()).encode(
+            "CoverageCollection", feature_type
+        )  # noqa: E501
 
-        encoder = Covjsonkit().encode("CoverageCollection", feature_type)
-
-        self.coverage = encoder.from_polytope(result)
+        if timeseries_type == "datetime":
+            self.coverage = encoder.from_polytope_step(result)
+        else:
+            self.coverage = encoder.from_polytope(result)
 
         return self.coverage
 
@@ -107,7 +114,6 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]:
         #   * enforcing strings are actually strings (e.g. type=fc)
 
         time = request.pop("time").replace(":", "")
-        request["date"] = request["date"] + "T" + time
 
         # TODO: not restricting certain keywords:
         #   * AREA, GRID
@@ -137,6 +143,11 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]:
 
             # List of individual values -> Union of Selects
             else:
+                if k == "date":
+                    dates = []
+                    for s in split:
+                        dates.append(pd.Timestamp(s + "T" + time))
+                    split = dates
                 base_shapes.append(shapes.Select(k, split))
 
         return base_shapes

polytope_mars/config.py

@@ -0,0 +1,20 @@
+from conflator import ConfigModel
+from polytope.options import Config
+
+
+class DatacubeConfig(ConfigModel):
+
+    type: str = "gribjump"
+    config: str = "config.yaml"
+    uri: str = "http://localhost:8000"
+
+
+class CovjsonKitConfig(ConfigModel):
+    param_db: str = "ecmwf"
+
+
+class PolytopeMarsConfig(ConfigModel):
+
+    datacube: DatacubeConfig = DatacubeConfig()
+    options: Config = Config()
+    coverageconfig: CovjsonKitConfig = CovjsonKitConfig()

polytope_mars/features/timeseries.py

@@ -6,8 +6,9 @@
 class TimeSeries(Feature):
     def __init__(self, config):
         assert config.pop("type") == "timeseries"
-        self.start_step = config.pop("start", None)
-        self.end_step = config.pop("end", None)
+        # self.start_step = config.pop("start", None)
+        # self.end_step = config.pop("end", None)
+        self.axis = config.pop("axis", [])
 
         self.points = config.pop("points", [])
 
@@ -27,11 +28,11 @@ def get_shapes(self):
                     for p in self.points
                 ],
             ),
-            shapes.Span("step", self.start_step, self.end_step),
+            # shapes.Span("step", self.start_step, self.end_step),
         ]
 
     def incompatible_keys(self):
-        return ["step", "levellist"]
+        return ["levellist"]
 
     def coverage_type(self):
         return "PointSeries"

polytope_mars/version.py

@@ -1 +1 @@
-__version__ = "0.0.4"
+__version__ = "0.0.5"

tests/test_time_series.py

@@ -20,11 +20,11 @@ def setup_method(self):
             "domain": "g",
             "param": "167",
             "number": "1/2/3/4/5",
+            "step": "0/3/6/9",
             "feature": {
                 "type": "timeseries",
                 "points": [[0.035149384216, 0.0]],
-                "start": 0,
-                "end": 9,
+                "axis": "step",
             },
         }