You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by fg...@apache.org on 2019/01/23 17:56:43 UTC
[incubator-sdap-ningesterpy] 20/29: new python processor which can
promote a variable into a global attribute
This is an automated email from the ASF dual-hosted git repository.
fgreg pushed a commit to branch v1.0.0-rc1
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ningesterpy.git
commit df5e890f35e5c3450c3c1feac7c4ab21775b3fd9
Author: Frank Greguska <fr...@jpl.nasa.gov>
AuthorDate: Tue Mar 6 16:11:01 2018 -0800
new python processor which can promote a variable into a global attribute
---
sdap/processors/__init__.py | 2 +
.../processors/promotevariabletoglobalattribute.py | 49 +++++++++++++++++++++
tests/datafiles/not_empty_wswm.nc | Bin 33119 -> 1041568 bytes
tests/processorchain_test.py | 33 ++++++++++++++
tests/promotevariabletoglobalattribute_test.py | 38 ++++++++++++++++
5 files changed, 122 insertions(+)
diff --git a/sdap/processors/__init__.py b/sdap/processors/__init__.py
index 8a48500..6d4a679 100644
--- a/sdap/processors/__init__.py
+++ b/sdap/processors/__init__.py
@@ -56,6 +56,7 @@ from sdap.processors.deleteunitaxis import DeleteUnitAxis
from sdap.processors.emptytilefilter import EmptyTileFilter
from sdap.processors.kelvintocelsius import KelvinToCelsius
from sdap.processors.normalizetimebeginningofmonth import NormalizeTimeBeginningOfMonth
+from sdap.processors.promotevariabletoglobalattribute import PromoteVariableToGlobalAttribute
from sdap.processors.regrid1x1 import Regrid1x1
from sdap.processors.subtract180longitude import Subtract180Longitude
from sdap.processors.tilereadingprocessor import GridReadingProcessor, SwathReadingProcessor, TimeSeriesReadingProcessor
@@ -70,6 +71,7 @@ INSTALLED_PROCESSORS = {
"EmptyTileFilter": EmptyTileFilter,
"KelvinToCelsius": KelvinToCelsius,
"NormalizeTimeBeginningOfMonth": NormalizeTimeBeginningOfMonth,
+ "PromoteVariableToGlobalAttribute": PromoteVariableToGlobalAttribute,
"Regrid1x1": Regrid1x1,
"Subtract180Longitude": Subtract180Longitude,
"GridReadingProcessor": GridReadingProcessor,
diff --git a/sdap/processors/promotevariabletoglobalattribute.py b/sdap/processors/promotevariabletoglobalattribute.py
new file mode 100644
index 0000000..11fa9d8
--- /dev/null
+++ b/sdap/processors/promotevariabletoglobalattribute.py
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import nexusproto.DataTile_pb2
+from netCDF4 import Dataset
+
+from sdap.processors import NexusTileProcessor
+
+
+class PromoteVariableToGlobalAttribute(NexusTileProcessor):
+
+ def __init__(self, attribute_name, variable_name, dimensioned_by, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ self.attribute_name = attribute_name
+ self.variable_name = variable_name
+ self.dimensioned_by = dimensioned_by
+
+ def process_nexus_tile(self, nexus_tile):
+ output_tile = nexusproto.DataTile_pb2.NexusTile()
+ output_tile.CopyFrom(nexus_tile)
+
+ file_path = output_tile.summary.granule
+ file_path = file_path[len('file:'):] if file_path.startswith('file:') else file_path
+
+ dimtoslice = {}
+ for dimension in output_tile.summary.section_spec.split(','):
+ name, start, stop = dimension.split(':')
+ dimtoslice[name] = slice(int(start), int(stop))
+
+ with Dataset(file_path) as ds:
+ new_attr = output_tile.summary.global_attributes.add()
+ new_attr.name = self.attribute_name
+ new_attr.values.extend(
+ [str(v) for v in ds[self.variable_name][[dimtoslice[dim] for dim in self.dimensioned_by]]])
+
+ yield output_tile
diff --git a/tests/datafiles/not_empty_wswm.nc b/tests/datafiles/not_empty_wswm.nc
index 772bbcb..ce0ebcc 100644
Binary files a/tests/datafiles/not_empty_wswm.nc and b/tests/datafiles/not_empty_wswm.nc differ
diff --git a/tests/processorchain_test.py b/tests/processorchain_test.py
index 7657ba6..d7b5dfa 100644
--- a/tests/processorchain_test.py
+++ b/tests/processorchain_test.py
@@ -130,6 +130,39 @@ class TestRunChainMethod(unittest.TestCase):
self.assertEqual(0, len(results))
+ def test_run_chain_promote_var(self):
+ processor_list = [
+ {'name': 'GridReadingProcessor',
+ 'config': {'latitude': 'lat',
+ 'longitude': 'lon',
+ 'time': 'time',
+ 'variable_to_read': 'analysed_sst'}},
+ {'name': 'EmptyTileFilter'},
+ {'name': 'KelvinToCelsius'},
+ {'name': 'PromoteVariableToGlobalAttribute',
+ 'config': {
+ 'attribute_name': 'time_i',
+ 'variable_name': 'time',
+ 'dimensioned_by': ['time']
+ }},
+ {'name': 'TileSummarizingProcessor'}
+ ]
+ processorchain = ProcessorChain(processor_list)
+
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'partial_empty_mur.nc4')
+
+ input_tile = nexusproto.NexusTile()
+ tile_summary = nexusproto.TileSummary()
+ tile_summary.granule = "file:%s" % test_file
+ tile_summary.section_spec = "time:0:1,lat:489:499,lon:0:10"
+ input_tile.summary.CopyFrom(tile_summary)
+
+ results = list(processorchain.process(input_tile))
+
+ self.assertEqual(1, len(results))
+ tile = results[0]
+ self.assertEqual("1104483600", tile.summary.global_attributes[0].values[0])
+
if __name__ == '__main__':
unittest.main()
diff --git a/tests/promotevariabletoglobalattribute_test.py b/tests/promotevariabletoglobalattribute_test.py
new file mode 100644
index 0000000..1fbe21d
--- /dev/null
+++ b/tests/promotevariabletoglobalattribute_test.py
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+
+from os import path
+
+import sdap.processors
+import nexusproto.DataTile_pb2
+
+
+class TestReadWSWMData(unittest.TestCase):
+
+ def test_read_not_empty_wswm(self):
+ test_file = path.join(path.dirname(__file__), 'datafiles', 'not_empty_wswm.nc')
+
+ promoter = sdap.processors.PromoteVariableToGlobalAttribute('rivid_i', 'rivid', ('rivid',))
+
+ input_tile = nexusproto.DataTile_pb2.NexusTile()
+ tile_summary = nexusproto.DataTile_pb2.TileSummary()
+ tile_summary.granule = "file:%s" % test_file
+ tile_summary.section_spec = "time:0:5832,rivid:0:1"
+ input_tile.summary.CopyFrom(tile_summary)
+
+ results = list(promoter.process(input_tile))
+
+ print(results)
\ No newline at end of file