You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by rk...@apache.org on 2024/01/17 16:27:28 UTC
(incubator-sdap-ingester) 01/01: SDAP-502: Gridded tile gen squeeze bug patch
This is an automated email from the ASF dual-hosted git repository.
rkk pushed a commit to branch SDAP-502
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit f7d8c670fdb4134fe2d427672bf8e5b509e75f22
Author: rileykk <ri...@jpl.nasa.gov>
AuthorDate: Wed Jan 17 08:27:18 2024 -0800
SDAP-502: Gridded tile gen squeeze bug patch
---
CHANGELOG.md | 1 +
.../GridMultiVariableReadingProcessor.py | 13 +++++++++++--
.../reading_processors/GridReadingProcessor.py | 21 +++++++++++++++++++--
3 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 929e6c4..01e5136 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- SDAP-501: Updated dependencies to remove `chardet`
### Fixed
- SDAP-488: Workaround to build issue on Apple Silicon (M1/M2). GI image build installs nexusproto through PyPI instead of building from source. A build arg `BUILD_NEXUSPROTO` was defined to allow building from source if desired/
+- SDAP-502: Fix for rare bug where gridded tiles generated from inputs where there is a dimension length where `dimensionLenght mod tileSliceLenght == 1` would cause tile generation to fail. This is because `np.squeeze` is used on the coordinate arrays, which, if the generated tile has only a single lat or lon, would squeeze the corresponding coordinate into a dimensionless array, which would raise an error down the line when `len` was called with it. Added a check for this case that bot [...]
### Security
## [1.1.0] - 2023-04-26
diff --git a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
index 035bb6b..bab04f6 100644
--- a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
+++ b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
@@ -55,8 +55,17 @@ class GridMultiVariableReadingProcessor(TileReadingProcessor):
lat_subset = ds[self.latitude][type(self)._slices_for_variable(ds[self.latitude], dimensions_to_slices)]
lon_subset = ds[self.longitude][type(self)._slices_for_variable(ds[self.longitude], dimensions_to_slices)]
- lat_subset = np.ma.filled(np.squeeze(lat_subset), np.NaN)
- lon_subset = np.ma.filled(np.squeeze(lon_subset), np.NaN)
+
+ lat_subset = np.squeeze(lat_subset)
+ if lat_subset.shape == ():
+ lat_subset = np.expand_dims(lat_subset, 0)
+
+ lon_subset = np.squeeze(lon_subset)
+ if lon_subset.shape == ():
+ lon_subset = np.expand_dims(lon_subset, 0)
+
+ lat_subset = np.ma.filled(lat_subset, np.NaN)
+ lon_subset = np.ma.filled(lon_subset, np.NaN)
if not isinstance(self.variable, list):
raise ValueError(f'self.variable `{self.variable}` needs to be a list. use GridReadingProcessor for single band Grid files.')
diff --git a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
index 73969e6..95bb7c6 100644
--- a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
+++ b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
@@ -36,15 +36,32 @@ class GridReadingProcessor(TileReadingProcessor):
data_variable = self.variable[0] if isinstance(self.variable, list) else self.variable
new_tile = nexusproto.GridTile()
+ expand_axes = []
+
lat_subset = ds[self.latitude][type(self)._slices_for_variable(ds[self.latitude], dimensions_to_slices)]
lon_subset = ds[self.longitude][type(self)._slices_for_variable(ds[self.longitude], dimensions_to_slices)]
- lat_subset = np.ma.filled(np.squeeze(lat_subset), np.NaN)
- lon_subset = np.ma.filled(np.squeeze(lon_subset), np.NaN)
+
+ lat_subset = np.squeeze(lat_subset)
+ if lat_subset.shape == ():
+ lat_subset = np.expand_dims(lat_subset, 0)
+ expand_axes.append(0)
+
+ lon_subset = np.squeeze(lon_subset)
+ if lon_subset.shape == ():
+ lon_subset = np.expand_dims(lon_subset, 0)
+ expand_axes.append(1)
+
+ lat_subset = np.ma.filled(lat_subset, np.NaN)
+ lon_subset = np.ma.filled(lon_subset, np.NaN)
data_subset = ds[data_variable][type(self)._slices_for_variable(ds[data_variable],
dimensions_to_slices)].data
data_subset = np.array(np.squeeze(data_subset))
+ if len(expand_axes) > 0:
+ data_subset = np.expand_dims(data_subset, tuple(expand_axes))
+
+
if self.depth:
depth_dim, depth_slice = list(type(self)._slices_for_variable(ds[self.depth],
dimensions_to_slices).items())[0]