You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by nc...@apache.org on 2021/05/07 00:18:34 UTC
[incubator-sdap-ingester] branch dev updated: SDAP-310: Added
standard_name to nexus tile and solr doc (#33)
This is an automated email from the ASF dual-hosted git repository.
nchung pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/dev by this push:
new 472ab15 SDAP-310: Added standard_name to nexus tile and solr doc (#33)
472ab15 is described below
commit 472ab158c5bcfd4001a92cc5ebb98c7827f146f8
Author: Stepheny Perez <sk...@users.noreply.github.com>
AuthorDate: Thu May 6 17:18:28 2021 -0700
SDAP-310: Added standard_name to nexus tile and solr doc (#33)
* Added standard_name to nexus tile and solr doc
* CDMS-69: Use standard_name in solr doc tile_var_name_s field if available
---
.../processors/TileSummarizingProcessor.py | 6 ++-
.../granule_ingester/writers/SolrStore.py | 4 +-
.../test_TileSummarizingProcessor.py | 44 ++++++++++++++++++++++
granule_ingester/tests/writers/test_SolrStore.py | 18 ++++++++-
4 files changed, 69 insertions(+), 3 deletions(-)
diff --git a/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py b/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
index 1fe5d7d..9f3c3e6 100644
--- a/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
+++ b/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
@@ -41,7 +41,7 @@ class TileSummarizingProcessor(TileProcessor):
super().__init__(*args, **kwargs)
self._dataset_name = dataset_name
- def process(self, tile, *args, **kwargs):
+ def process(self, tile, dataset, *args, **kwargs):
tile_type = tile.tile.WhichOneof("tile_type")
tile_data = getattr(tile.tile, tile_type)
@@ -81,6 +81,10 @@ class TileSummarizingProcessor(TileProcessor):
except NoTimeException:
pass
+ standard_name = dataset.variables[tile_summary.data_var_name].attrs.get('standard_name')
+ if standard_name:
+ tile_summary.standard_name = standard_name
+
tile.summary.CopyFrom(tile_summary)
return tile
diff --git a/granule_ingester/granule_ingester/writers/SolrStore.py b/granule_ingester/granule_ingester/writers/SolrStore.py
index 67532b5..b753404 100644
--- a/granule_ingester/granule_ingester/writers/SolrStore.py
+++ b/granule_ingester/granule_ingester/writers/SolrStore.py
@@ -103,6 +103,8 @@ class SolrStore(MetadataStore):
tile_type = tile.tile.WhichOneof("tile_type")
tile_data = getattr(tile.tile, tile_type)
+ var_name = summary.standard_name if summary.standard_name else summary.data_var_name
+
input_document = {
'table_s': self.TABLE_NAME,
'geo': geo,
@@ -111,7 +113,7 @@ class SolrStore(MetadataStore):
'sectionSpec_s': summary.section_spec,
'dataset_s': summary.dataset_name,
'granule_s': granule_file_name,
- 'tile_var_name_s': summary.data_var_name,
+ 'tile_var_name_s': var_name,
'tile_min_lon': bbox.lon_min,
'tile_max_lon': bbox.lon_max,
'tile_min_lat': bbox.lat_min,
diff --git a/granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py b/granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py
new file mode 100644
index 0000000..a0b1ece
--- /dev/null
+++ b/granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py
@@ -0,0 +1,44 @@
+import unittest
+from os import path
+
+import xarray as xr
+from granule_ingester.processors import TileSummarizingProcessor
+from granule_ingester.processors.reading_processors.GridReadingProcessor import GridReadingProcessor
+from nexusproto import DataTile_pb2 as nexusproto
+
+
+class TestTileSummarizingProcessor(unittest.TestCase):
+ def test_standard_name_exists(self):
+ """
+ Test that the standard_name attribute exists in a
+ Tile.TileSummary object after being processed with
+ TileSummarizingProcessor
+ """
+ reading_processor = GridReadingProcessor(
+ variable='analysed_sst',
+ latitude='lat',
+ longitude='lon',
+ time='time',
+ tile='tile'
+ )
+ relative_path = '../granules/20050101120000-NCEI-L4_GHRSST-SSTblend-AVHRR_OI-GLOB-v02.0-fv02.0.nc'
+ granule_path = path.join(path.dirname(__file__), relative_path)
+ tile_summary = nexusproto.TileSummary()
+ tile_summary.granule = granule_path
+ tile_summary.data_var_name = 'analysed_sst'
+
+ input_tile = nexusproto.NexusTile()
+ input_tile.summary.CopyFrom(tile_summary)
+
+ dims = {
+ 'lat': slice(0, 30),
+ 'lon': slice(0, 30),
+ 'time': slice(0, 1),
+ 'tile': slice(10, 11),
+ }
+
+ with xr.open_dataset(granule_path, decode_cf=True) as ds:
+ output_tile = reading_processor._generate_tile(ds, dims, input_tile)
+ tile_summary_processor = TileSummarizingProcessor('test')
+ new_tile = tile_summary_processor.process(tile=output_tile, dataset=ds)
+ assert new_tile.summary.standard_name == 'sea_surface_temperature'
diff --git a/granule_ingester/tests/writers/test_SolrStore.py b/granule_ingester/tests/writers/test_SolrStore.py
index 0e971ce..2225f42 100644
--- a/granule_ingester/tests/writers/test_SolrStore.py
+++ b/granule_ingester/tests/writers/test_SolrStore.py
@@ -25,6 +25,7 @@ class TestSolrStore(unittest.TestCase):
tile.summary.stats.count = 100
tile.summary.stats.min_time = 694224000
tile.summary.stats.max_time = 694310400
+ tile.summary.standard_name = 'sea_surface_temperature'
tile.tile.ecco_tile.depth = 10.5
@@ -39,7 +40,7 @@ class TestSolrStore(unittest.TestCase):
self.assertEqual('test_dataset!test_id', solr_doc['solr_id_s'])
self.assertEqual('time:0:1,j:0:20,i:200:240', solr_doc['sectionSpec_s'])
self.assertEqual('test_granule_path', solr_doc['granule_s'])
- self.assertEqual('test_variable', solr_doc['tile_var_name_s'])
+ self.assertEqual('sea_surface_temperature', solr_doc['tile_var_name_s'])
self.assertAlmostEqual(-90.5, solr_doc['tile_min_lon'])
self.assertAlmostEqual(90.0, solr_doc['tile_max_lon'])
self.assertAlmostEqual(-180.1, solr_doc['tile_min_lat'], delta=1E-5)
@@ -51,3 +52,18 @@ class TestSolrStore(unittest.TestCase):
self.assertAlmostEqual(12.5, solr_doc['tile_avg_val_d'])
self.assertEqual(100, solr_doc['tile_count_i'])
self.assertAlmostEqual(10.5, solr_doc['tile_depth'])
+
+ def test_build_solr_doc_no_standard_name(self):
+ """
+ When TileSummary.standard_name isn't available, the solr field
+ tile_var_name_s should use TileSummary.data_var_name
+ """
+ tile = nexusproto.NexusTile()
+ tile.summary.tile_id = 'test_id'
+ tile.summary.data_var_name = 'test_variable'
+ tile.tile.ecco_tile.depth = 10.5
+
+ metadata_store = SolrStore()
+ solr_doc = metadata_store._build_solr_doc(tile)
+
+ self.assertEqual('test_variable', solr_doc['tile_var_name_s'])