You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/11/18 22:12:08 UTC
[incubator-sdap-ingester] branch dev updated: SDAP-293: Automatic
Kelvin-to-Celsius conversion (#22)
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/dev by this push:
new f66c7e1 SDAP-293: Automatic Kelvin-to-Celsius conversion (#22)
f66c7e1 is described below
commit f66c7e1614d1bdf63c09ba14cd850c893e2db3d3
Author: Elizabeth Yam <ec...@users.noreply.github.com>
AuthorDate: Wed Nov 18 14:11:58 2020 -0800
SDAP-293: Automatic Kelvin-to-Celsius conversion (#22)
Co-authored-by: Eamon Ford <ea...@gmail.com>
---
.../services/CollectionProcessor.py | 1 +
granule_ingester/granule_ingester/README.md | 11 +++++++++++
.../granule_ingester/processors/TileProcessor.py | 20 +++++++++++++++++++-
.../granule_ingester/processors/kelvintocelsius.py | 18 +++++++++++-------
4 files changed, 42 insertions(+), 8 deletions(-)
diff --git a/collection_manager/collection_manager/services/CollectionProcessor.py b/collection_manager/collection_manager/services/CollectionProcessor.py
index 96c461e..bab56fc 100644
--- a/collection_manager/collection_manager/services/CollectionProcessor.py
+++ b/collection_manager/collection_manager/services/CollectionProcessor.py
@@ -87,6 +87,7 @@ class CollectionProcessor:
},
{'name': 'emptyTileFilter'},
{'name': 'subtract180FromLongitude'},
+ {'name': 'kelvinToCelsius'},
{
'name': 'tileSummary',
'dataset_name': collection.dataset_id
diff --git a/granule_ingester/granule_ingester/README.md b/granule_ingester/granule_ingester/README.md
new file mode 100644
index 0000000..881461a
--- /dev/null
+++ b/granule_ingester/granule_ingester/README.md
@@ -0,0 +1,11 @@
+## Plugin Processor Architecture
+The operator can write a class that inherits from the `TileProcessor` class and implements the abstract function `process`, which among other things, takes in the NexusTile(`nexusproto.DataTile_pb2.NexusTile` object)) generated with default configurations and the NC4 Dataset(`xarray.Dataset` object), and allows the user to add further modifications to how granule data is saved.
+
+Any additional transformation the operator needs to accomplish must be done in this `process` method, which is what is ultimately called in the ingestion pipeline. Helper functions are suggested for breaking up complex procedures.
+
+The custom code file would be copied into the ingestion pods via the helm chart (see chart for local and mount paths).
+
+Example: `KelvinToCelsiusProcessor`
+This processor checks the units of the saved variable. If it is some form of Kelvin, it automatically converts all of the temperature measurements to Celsius by subtracting 273.15 from each data point. The transformed data then replaces the default (untransformed) values and the processor returns the modified tile.
+
+#### TODO Add configuration option for unusual representations of temperature units.
\ No newline at end of file
diff --git a/granule_ingester/granule_ingester/processors/TileProcessor.py b/granule_ingester/granule_ingester/processors/TileProcessor.py
index d62c504..47b1a2f 100644
--- a/granule_ingester/granule_ingester/processors/TileProcessor.py
+++ b/granule_ingester/granule_ingester/processors/TileProcessor.py
@@ -14,10 +14,28 @@
# limitations under the License.
from abc import ABC, abstractmethod
+from nexusproto.serialization import from_shaped_array, to_shaped_array
+from nexusproto.DataTile_pb2 import NexusTile
# TODO: make this an informal interface, not an abstract class
class TileProcessor(ABC):
@abstractmethod
- def process(self, tile, *args, **kwargs):
+ def process(self, tile: NexusTile, *args, **kwargs):
+ # accessing the data
+ # the_tile_type = tile.tile.WhichOneof("tile_type")
+ # the_tile_data = getattr(tile.tile, the_tile_type)
+
+ # get netCDF as xarray.Dataset object
+ # ds = kwargs['dataset']
+
+ # example transformation:
+ # var_data = from_shaped_array(the_tile_data.variable_data) - 273.15
+
+ # save transformed data back into tile
+ # the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data))
+
+ # return transformed tile
+ # return tile
+
pass
diff --git a/granule_ingester/granule_ingester/processors/kelvintocelsius.py b/granule_ingester/granule_ingester/processors/kelvintocelsius.py
index e728418..9ad4f49 100644
--- a/granule_ingester/granule_ingester/processors/kelvintocelsius.py
+++ b/granule_ingester/granule_ingester/processors/kelvintocelsius.py
@@ -13,19 +13,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
from nexusproto.serialization import from_shaped_array, to_shaped_array
-
+from nexusproto.DataTile_pb2 import NexusTile
from granule_ingester.processors.TileProcessor import TileProcessor
class KelvinToCelsius(TileProcessor):
- def process(self, tile, *args, **kwargs):
+ def process(self, tile: NexusTile, *args, **kwargs):
the_tile_type = tile.tile.WhichOneof("tile_type")
the_tile_data = getattr(tile.tile, the_tile_type)
-
- var_data = from_shaped_array(the_tile_data.variable_data) - 273.15
-
- the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data))
+ kelvins = ['kelvin', 'degk', 'deg_k', 'degreesk', 'degrees_k', 'degree_k', 'degreek']
+
+ if 'dataset' in kwargs:
+ ds = kwargs['dataset']
+ variable_name = tile.summary.data_var_name
+ variable_unit = ds.variables[variable_name].attrs['units']
+ if any([unit in variable_unit.lower() for unit in kelvins]):
+ var_data = from_shaped_array(the_tile_data.variable_data) - 273.15
+ the_tile_data.variable_data.CopyFrom(to_shaped_array(var_data))
return tile