You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by rk...@apache.org on 2024/01/17 16:27:27 UTC
(incubator-sdap-ingester) branch SDAP-502 created (now f7d8c67)
This is an automated email from the ASF dual-hosted git repository.
rkk pushed a change to branch SDAP-502
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
at f7d8c67 SDAP-502: Gridded tile gen squeeze bug patch
This branch includes the following new commits:
new f7d8c67 SDAP-502: Gridded tile gen squeeze bug patch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-sdap-ingester) 01/01: SDAP-502: Gridded tile gen squeeze bug patch
Posted by rk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rkk pushed a commit to branch SDAP-502
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit f7d8c670fdb4134fe2d427672bf8e5b509e75f22
Author: rileykk <ri...@jpl.nasa.gov>
AuthorDate: Wed Jan 17 08:27:18 2024 -0800
SDAP-502: Gridded tile gen squeeze bug patch
---
CHANGELOG.md | 1 +
.../GridMultiVariableReadingProcessor.py | 13 +++++++++++--
.../reading_processors/GridReadingProcessor.py | 21 +++++++++++++++++++--
3 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 929e6c4..01e5136 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- SDAP-501: Updated dependencies to remove `chardet`
### Fixed
- SDAP-488: Workaround to build issue on Apple Silicon (M1/M2). GI image build installs nexusproto through PyPI instead of building from source. A build arg `BUILD_NEXUSPROTO` was defined to allow building from source if desired/
+- SDAP-502: Fix for rare bug where gridded tiles generated from inputs where there is a dimension length where `dimensionLenght mod tileSliceLenght == 1` would cause tile generation to fail. This is because `np.squeeze` is used on the coordinate arrays, which, if the generated tile has only a single lat or lon, would squeeze the corresponding coordinate into a dimensionless array, which would raise an error down the line when `len` was called with it. Added a check for this case that bot [...]
### Security
## [1.1.0] - 2023-04-26
diff --git a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
index 035bb6b..bab04f6 100644
--- a/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
+++ b/granule_ingester/granule_ingester/processors/reading_processors/GridMultiVariableReadingProcessor.py
@@ -55,8 +55,17 @@ class GridMultiVariableReadingProcessor(TileReadingProcessor):
lat_subset = ds[self.latitude][type(self)._slices_for_variable(ds[self.latitude], dimensions_to_slices)]
lon_subset = ds[self.longitude][type(self)._slices_for_variable(ds[self.longitude], dimensions_to_slices)]
- lat_subset = np.ma.filled(np.squeeze(lat_subset), np.NaN)
- lon_subset = np.ma.filled(np.squeeze(lon_subset), np.NaN)
+
+ lat_subset = np.squeeze(lat_subset)
+ if lat_subset.shape == ():
+ lat_subset = np.expand_dims(lat_subset, 0)
+
+ lon_subset = np.squeeze(lon_subset)
+ if lon_subset.shape == ():
+ lon_subset = np.expand_dims(lon_subset, 0)
+
+ lat_subset = np.ma.filled(lat_subset, np.NaN)
+ lon_subset = np.ma.filled(lon_subset, np.NaN)
if not isinstance(self.variable, list):
raise ValueError(f'self.variable `{self.variable}` needs to be a list. use GridReadingProcessor for single band Grid files.')
diff --git a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
index 73969e6..95bb7c6 100644
--- a/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
+++ b/granule_ingester/granule_ingester/processors/reading_processors/GridReadingProcessor.py
@@ -36,15 +36,32 @@ class GridReadingProcessor(TileReadingProcessor):
data_variable = self.variable[0] if isinstance(self.variable, list) else self.variable
new_tile = nexusproto.GridTile()
+ expand_axes = []
+
lat_subset = ds[self.latitude][type(self)._slices_for_variable(ds[self.latitude], dimensions_to_slices)]
lon_subset = ds[self.longitude][type(self)._slices_for_variable(ds[self.longitude], dimensions_to_slices)]
- lat_subset = np.ma.filled(np.squeeze(lat_subset), np.NaN)
- lon_subset = np.ma.filled(np.squeeze(lon_subset), np.NaN)
+
+ lat_subset = np.squeeze(lat_subset)
+ if lat_subset.shape == ():
+ lat_subset = np.expand_dims(lat_subset, 0)
+ expand_axes.append(0)
+
+ lon_subset = np.squeeze(lon_subset)
+ if lon_subset.shape == ():
+ lon_subset = np.expand_dims(lon_subset, 0)
+ expand_axes.append(1)
+
+ lat_subset = np.ma.filled(lat_subset, np.NaN)
+ lon_subset = np.ma.filled(lon_subset, np.NaN)
data_subset = ds[data_variable][type(self)._slices_for_variable(ds[data_variable],
dimensions_to_slices)].data
data_subset = np.array(np.squeeze(data_subset))
+ if len(expand_axes) > 0:
+ data_subset = np.expand_dims(data_subset, tuple(expand_axes))
+
+
if self.depth:
depth_dim, depth_slice = list(type(self)._slices_for_variable(ds[self.depth],
dimensions_to_slices).items())[0]