You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by rk...@apache.org on 2024/07/09 20:58:18 UTC
(sdap-ingester) branch SDAP-523 created (now be49d26)
This is an automated email from the ASF dual-hosted git repository.
rkk pushed a change to branch SDAP-523
in repository https://gitbox.apache.org/repos/asf/sdap-ingester.git
at be49d26 Fixes for test case failures and excessive warnings
This branch includes the following new commits:
new be49d26 Fixes for test case failures and excessive warnings
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(sdap-ingester) 01/01: Fixes for test case failures and excessive warnings
Posted by rk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rkk pushed a commit to branch SDAP-523
in repository https://gitbox.apache.org/repos/asf/sdap-ingester.git
commit be49d26d618ce46d8a0b9b04dad195f146679126
Author: rileykk <ri...@jpl.nasa.gov>
AuthorDate: Tue Jul 9 13:57:54 2024 -0700
Fixes for test case failures and excessive warnings
---
CHANGELOG.md | 9 +++++++
.../processors/TileSummarizingProcessor.py | 9 +++++--
.../processors/test_ForceAscendingLatitude.py | 31 ++++++++++++----------
.../test_TileSummarizingProcessor.py | 6 ++---
.../test_GridMultiBandReadingProcessor.py | 17 +++++++-----
.../test_GridReadingProcessor.py | 17 +++++++-----
.../tests/slicers/test_SliceFileByStepSize.py | 10 ++++++-
granule_ingester/tests/writers/test_SolrStore.py | 18 ++++++++-----
8 files changed, 77 insertions(+), 40 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 78a71db..6dad868 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [Unreleased] - Unreleased
+### Added
+### Changed
+### Deprecated
+### Removed
+### Fixed
+- SDAP-523: Fixes for test case failures and excessive warnings in Ingester unit tests
+### Security
+
## [1.3.0] - 2024-06-10
### Added
- SDAP-472: Added support for defining Zarr collections in the collection config
diff --git a/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py b/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
index 041cac5..9eba094 100644
--- a/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
+++ b/granule_ingester/granule_ingester/processors/TileSummarizingProcessor.py
@@ -62,8 +62,13 @@ class TileSummarizingProcessor(TileProcessor):
tile_summary.bbox.lat_max = numpy.nanmax(latitudes).item()
tile_summary.bbox.lon_min = numpy.nanmin(longitudes).item()
tile_summary.bbox.lon_max = numpy.nanmax(longitudes).item()
- tile_summary.stats.min = numpy.nanmin(data).item()
- tile_summary.stats.max = numpy.nanmax(data).item()
+
+ if all(numpy.isnan(data).flatten()):
+ tile_summary.stats.min = numpy.nan
+ tile_summary.stats.max = numpy.nan
+ else:
+ tile_summary.stats.min = numpy.nanmin(data).item()
+ tile_summary.stats.max = numpy.nanmax(data).item()
tile_summary.stats.count = data.size - numpy.count_nonzero(numpy.isnan(data))
logger.debug(f'set summary fields')
diff --git a/granule_ingester/tests/processors/test_ForceAscendingLatitude.py b/granule_ingester/tests/processors/test_ForceAscendingLatitude.py
index 0d16d81..38a48ec 100644
--- a/granule_ingester/tests/processors/test_ForceAscendingLatitude.py
+++ b/granule_ingester/tests/processors/test_ForceAscendingLatitude.py
@@ -18,6 +18,7 @@ import unittest
import xarray as xr
import numpy as np
from os import path
+import warnings
from granule_ingester.processors.reading_processors import GridMultiVariableReadingProcessor, GridReadingProcessor
from nexusproto import DataTile_pb2 as nexusproto
@@ -41,20 +42,22 @@ class TestForceAscendingLatitude(unittest.TestCase):
'lon': slice(0, 30)
}
- with xr.open_dataset(granule_path) as ds:
- tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
- flipped_tile = ForceAscendingLatitude().process(tile)
- the_flipped_tile_type = flipped_tile.tile.WhichOneof("tile_type")
- self.assertEqual(the_flipped_tile_type, 'grid_multi_variable_tile', f'wrong tile type')
- the_flipped_tile_data = getattr(flipped_tile.tile, the_flipped_tile_type)
- self.assertEqual([1, 30, 30, 2], the_flipped_tile_data.variable_data.shape)
- flipped_latitudes = from_shaped_array(the_flipped_tile_data.latitude)
- original_lat_data = ds['lat'].values
- np.testing.assert_almost_equal(flipped_latitudes[0], original_lat_data[29], decimal=5, err_msg='wrong first vs last latitude', verbose=True)
- np.testing.assert_almost_equal(flipped_latitudes[1], original_lat_data[28], decimal=5, err_msg='wrong latitude', verbose=True)
- flipped_data = from_shaped_array(the_flipped_tile_data.variable_data)
- original_b04_data = ds['B04'].values
- np.testing.assert_almost_equal(original_b04_data[0][0][0], flipped_data[0][29][0][1], decimal=4, err_msg='wrong first vs last data', verbose=True)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', RuntimeWarning)
+ with xr.open_dataset(granule_path) as ds:
+ tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
+ flipped_tile = ForceAscendingLatitude().process(tile)
+ the_flipped_tile_type = flipped_tile.tile.WhichOneof("tile_type")
+ self.assertEqual(the_flipped_tile_type, 'grid_multi_variable_tile', f'wrong tile type')
+ the_flipped_tile_data = getattr(flipped_tile.tile, the_flipped_tile_type)
+ self.assertEqual([1, 30, 30, 2], the_flipped_tile_data.variable_data.shape)
+ flipped_latitudes = from_shaped_array(the_flipped_tile_data.latitude)
+ original_lat_data = ds['lat'].values
+ np.testing.assert_almost_equal(flipped_latitudes[0], original_lat_data[29], decimal=5, err_msg='wrong first vs last latitude', verbose=True)
+ np.testing.assert_almost_equal(flipped_latitudes[1], original_lat_data[28], decimal=5, err_msg='wrong latitude', verbose=True)
+ flipped_data = from_shaped_array(the_flipped_tile_data.variable_data)
+ original_b04_data = ds['B04'].values
+ np.testing.assert_almost_equal(original_b04_data[0][0][0], flipped_data[0][29][0][1], decimal=4, err_msg='wrong first vs last data', verbose=True)
return
def test_02_grid_single_band_data(self):
diff --git a/granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py b/granule_ingester/tests/processors/test_TileSummarizingProcessor.py
similarity index 93%
rename from granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py
rename to granule_ingester/tests/processors/test_TileSummarizingProcessor.py
index a0ac782..2d2641a 100644
--- a/granule_ingester/tests/reading_processors/test_TileSummarizingProcessor.py
+++ b/granule_ingester/tests/processors/test_TileSummarizingProcessor.py
@@ -58,7 +58,7 @@ class TestTileSummarizingProcessor(unittest.TestCase):
output_tile = reading_processor._generate_tile(ds, dims, input_tile)
tile_summary_processor = TileSummarizingProcessor('test')
new_tile = tile_summary_processor.process(tile=output_tile, dataset=ds)
- self.assertEqual('"sea_surface_temperature"', new_tile.summary.standard_name, f'wrong new_tile.summary.standard_name')
+ self.assertEqual('["sea_surface_temperature"]', new_tile.summary.standard_name, f'wrong new_tile.summary.standard_name')
def test_hls_single_var01(self):
"""
@@ -89,8 +89,8 @@ class TestTileSummarizingProcessor(unittest.TestCase):
output_tile = reading_processor._generate_tile(ds, dimensions_to_slices, input_tile)
tile_summary_processor = TileSummarizingProcessor('test')
new_tile = tile_summary_processor.process(tile=output_tile, dataset=ds)
- self.assertEqual('null', new_tile.summary.standard_name, f'wrong new_tile.summary.standard_name')
- self.assertEqual(None, json.loads(new_tile.summary.standard_name), f'unable to convert new_tile.summary.standard_name from JSON')
+ self.assertEqual('[null]', new_tile.summary.standard_name, f'wrong new_tile.summary.standard_name')
+ self.assertEqual([None], json.loads(new_tile.summary.standard_name), f'unable to convert new_tile.summary.standard_name from JSON')
self.assertTrue(abs(new_tile.summary.stats.mean - 0.26137) < 0.001, f'mean value is not close expected: 0.26137. actual: {new_tile.summary.stats.mean}')
def test_hls_multiple_var_01(self):
diff --git a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
index 3c47718..291028a 100644
--- a/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
+++ b/granule_ingester/tests/reading_processors/test_GridMultiBandReadingProcessor.py
@@ -14,6 +14,7 @@
# limitations under the License.
import json
import unittest
+import warnings
from os import path
import numpy as np
@@ -185,13 +186,15 @@ class TestCalendars(unittest.TestCase):
tile = nexusproto.NexusTile()
with xr.open_dataset(granule_path, decode_cf=True) as ds:
- reading_processor = GridMultiVariableReadingProcessor(
- data_vars,
- lat_var,
- lon_var,
- time=time_var
- )
- tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', RuntimeWarning)
+ reading_processor = GridMultiVariableReadingProcessor(
+ data_vars,
+ lat_var,
+ lon_var,
+ time=time_var
+ )
+ tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile)
assert tile.tile.grid_multi_variable_tile.time
def test_julian_calendar_tile(self):
diff --git a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
index 3123cc4..a0798fe 100644
--- a/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
+++ b/granule_ingester/tests/reading_processors/test_GridReadingProcessor.py
@@ -14,6 +14,7 @@
# limitations under the License.
import unittest
+import warnings
from os import path
import numpy as np
@@ -422,13 +423,15 @@ class TestCalendars(unittest.TestCase):
tile = nexusproto.NexusTile()
with xr.open_dataset(granule_path, decode_cf=True) as ds:
- reading_processor = GridReadingProcessor(
- [data_var],
- lat_var,
- lon_var,
- time=time_var
- )
- tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile)
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', RuntimeWarning)
+ reading_processor = GridReadingProcessor(
+ [data_var],
+ lat_var,
+ lon_var,
+ time=time_var
+ )
+ tile = reading_processor._generate_tile(ds, dimensions_to_slices, tile)
assert tile.tile.grid_tile.time
def test_julian_calendar_tile(self):
diff --git a/granule_ingester/tests/slicers/test_SliceFileByStepSize.py b/granule_ingester/tests/slicers/test_SliceFileByStepSize.py
index 5c091c1..81d032d 100644
--- a/granule_ingester/tests/slicers/test_SliceFileByStepSize.py
+++ b/granule_ingester/tests/slicers/test_SliceFileByStepSize.py
@@ -48,7 +48,15 @@ class TestSliceFileByStepSize(unittest.TestCase):
'depth:2:4,latitude:180:360,longitude:540:720,nv:0:2,time:0:1'
]
- self.assertEqual(expected_slices, slices)
+ # Neither the order of slices, nor the order of dimensions within the slices matter
+
+ self.assertEqual(len(expected_slices), len(slices))
+
+ def slice_spec_to_tuples(s):
+ return tuple([(name, slc) for name, slc in sorted([dim.split(':', 1) for dim in s.split(',')])])
+
+ self.assertSetEqual(set([slice_spec_to_tuples(ex) for ex in expected_slices]),
+ set([slice_spec_to_tuples(s) for s in slices]))
def test_generate_slices_indexed(self):
netcdf_path = path.join(path.dirname(__file__), '../granules/SMAP_L2B_SSS_04892_20160101T005507_R13080.h5')
diff --git a/granule_ingester/tests/writers/test_SolrStore.py b/granule_ingester/tests/writers/test_SolrStore.py
index 95a163a..2160c92 100644
--- a/granule_ingester/tests/writers/test_SolrStore.py
+++ b/granule_ingester/tests/writers/test_SolrStore.py
@@ -43,7 +43,8 @@ class TestSolrStore(unittest.TestCase):
tile.summary.stats.max_time = 694310400
tile.summary.standard_name = json.dumps('sea_surface_temperature')
- tile.tile.ecco_tile.depth = 10.5
+ tile.tile.grid_tile.min_elevation = -10.5
+ tile.tile.grid_tile.max_elevation = 0
metadata_store = SolrStore()
solr_doc = metadata_store._build_solr_doc(tile)
@@ -68,7 +69,8 @@ class TestSolrStore(unittest.TestCase):
self.assertAlmostEqual(25.5, solr_doc['tile_max_val_d'])
self.assertAlmostEqual(12.5, solr_doc['tile_avg_val_d'])
self.assertEqual(100, solr_doc['tile_count_i'])
- self.assertAlmostEqual(10.5, solr_doc['tile_depth'])
+ self.assertAlmostEqual(-10.5, solr_doc['tile_min_elevation_d'])
+ self.assertAlmostEqual(0, solr_doc['tile_max_elevation_d'])
def test_build_solr_doc_no_standard_name_02(self):
tile = nexusproto.NexusTile()
@@ -89,7 +91,8 @@ class TestSolrStore(unittest.TestCase):
tile.summary.stats.min_time = 694224000
tile.summary.stats.max_time = 694310400
- tile.tile.ecco_tile.depth = 10.5
+ tile.tile.grid_tile.min_elevation = -10.5
+ tile.tile.grid_tile.max_elevation = 0
metadata_store = SolrStore()
solr_doc = metadata_store._build_solr_doc(tile)
@@ -113,7 +116,8 @@ class TestSolrStore(unittest.TestCase):
self.assertAlmostEqual(25.5, solr_doc['tile_max_val_d'])
self.assertAlmostEqual(12.5, solr_doc['tile_avg_val_d'])
self.assertEqual(100, solr_doc['tile_count_i'])
- self.assertAlmostEqual(10.5, solr_doc['tile_depth'])
+ self.assertAlmostEqual(-10.5, solr_doc['tile_min_elevation_d'])
+ self.assertAlmostEqual(0, solr_doc['tile_max_elevation_d'])
def test_build_solr_doc_no_standard_name(self):
"""
@@ -123,7 +127,8 @@ class TestSolrStore(unittest.TestCase):
tile = nexusproto.NexusTile()
tile.summary.tile_id = 'test_id'
tile.summary.data_var_name = json.dumps(['test_variable', 'test_variable_02'])
- tile.tile.ecco_tile.depth = 10.5
+ tile.tile.grid_tile.min_elevation = -10.5
+ tile.tile.grid_tile.max_elevation = 0
metadata_store = SolrStore()
solr_doc = metadata_store._build_solr_doc(tile)
@@ -142,7 +147,8 @@ class TestSolrStore(unittest.TestCase):
tile.summary.tile_id = 'test_id'
tile.summary.data_var_name = json.dumps(['test_variable', 'test_variable_02'])
tile.summary.standard_name = json.dumps(['sea_surface_temperature', None])
- tile.tile.ecco_tile.depth = 10.5
+ tile.tile.grid_tile.min_elevation = -10.5
+ tile.tile.grid_tile.max_elevation = 0
metadata_store = SolrStore()
solr_doc = metadata_store._build_solr_doc(tile)