You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by sk...@apache.org on 2023/03/20 18:22:21 UTC
[incubator-sdap-nexus] branch master updated: SDAP-415: Satellite to satellite queries fail if using an L2 dataset (#235)
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
The following commit(s) were added to refs/heads/master by this push:
new cadf9a4 SDAP-415: Satellite to satellite queries fail if using an L2 dataset (#235)
cadf9a4 is described below
commit cadf9a4d664f198141310f4291427a7c86226362
Author: Stepheny Perez <sk...@users.noreply.github.com>
AuthorDate: Mon Mar 20 11:22:15 2023 -0700
SDAP-415: Satellite to satellite queries fail if using an L2 dataset (#235)
* Guide for building docker images
* TOC & refs
* Verify step
* Docker requirement
* Switched image repos to generic sdap instead of our nexuxjpl
* changelog
* Explicitly defined equality for DomsPoint.
This prevents the duplicate primary points from appearing in the final results by merging them in the combineByKey step.
* Revert "Explicitly defined equality for DomsPoint."
This reverts commit 3c01a9958d6ca825c9b794e59e69ba31108e3a05.
* Pointed quickstart guide to use ASF images for CM & GI
* Tarball extraction
* Pointed webapp image in qs to ASF repo
* Changelog
* Split version vars to reflect release artifacts
* Language: This guide is for SDAP not NEXUS
* Renamed build directory
* Pointed remaining sdap images to ASF repos
* Renamed solr image to solr cloud to better match public image name
* Changed repository link to ASF instead of JPL
* Update changelog
* Improved qs interoperability w/ locally built images
* Improved qs interoperability w/ locally built images
* SDAP-422: Update build guide extraction section to reflect updated release structure
* Update build.rst
* Don't simplify time_data arrays, this seems to cause issues down the line
* Restored simplification, added case for desired_shape[0] == 1 and multi_var
* Changelog
* Simplify _to_standard_index (#7)
Co-authored-by: skorper <st...@gmail.com>
* fixed sat to sat matchup bug
---------
Co-authored-by: rileykk <ri...@jpl.nasa.gov>
Co-authored-by: Riley Kuttruff <72...@users.noreply.github.com>
---
CHANGELOG.md | 1 +
analysis/webservice/algorithms_spark/Matchup.py | 27 ++++++++++++++++---------
data-access/nexustiles/model/nexusmodel.py | 10 +++------
3 files changed, 21 insertions(+), 17 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0aa4b91..b55c8c7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- SDAP-436: Added special case for handling Cassandra SwathMulti tiles with uniform time arrays
- SDAP-449: Fixed `/cdmsresults` NetCDF output displaying and downloading as .txt.
- SDAP-449: Fixed 404 error when populating datasets; script was still using `/domslist`
+- SDAP-415: Fixed bug where mask was incorrectly combined across all variables for multi-variable satellite to satellite matchup
### Security
## [1.0.0] - 2022-12-05
diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index 5c5ec74..2bc91c3 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -169,10 +169,11 @@ class Matchup(NexusCalcSparkHandler):
raise NexusProcessingException(reason="'secondary' argument is required", code=400)
parameter_s = request.get_argument('parameter')
- insitu_params = get_insitu_params(insitu_schema.get())
- if parameter_s and parameter_s not in insitu_params:
- raise NexusProcessingException(
- reason=f"Parameter {parameter_s} not supported. Must be one of {insitu_params}", code=400)
+ if parameter_s:
+ insitu_params = get_insitu_params(insitu_schema.get())
+ if parameter_s not in insitu_params:
+ raise NexusProcessingException(
+ reason=f"Parameter {parameter_s} not supported. Must be one of {insitu_params}", code=400)
try:
start_time = request.get_start_datetime()
@@ -801,31 +802,37 @@ def match_satellite_to_insitu(tile_ids, primary_b, secondary_b, parameter_b, tt_
polygon = Polygon(
[(west, south), (east, south), (east, north), (west, north), (west, south)])
+ # Find tile IDS from spatial/temporal bounds of partition
matchup_tiles = tile_service.find_tiles_in_polygon(
bounding_polygon=polygon,
ds=secondary_b.value,
start_time=matchup_min_time,
end_time=matchup_max_time,
- fetch_data=True,
+ fl='id',
+ fetch_data=False,
sort=['tile_min_time_dt asc', 'tile_min_lon asc', 'tile_min_lat asc'],
rows=5000
)
# Convert Tile IDS to tiles and convert to UTM lat/lon projection.
matchup_points = []
+ edge_results = []
for tile in matchup_tiles:
+ # Retrieve tile data and convert to lat/lon projection
+ tiles = tile_service.find_tile_by_id(tile.tile_id, fetch_data=True)
+ tile = tiles[0]
+
valid_indices = tile.get_indices()
+
primary_points = np.array([aeqd_proj(
tile.longitudes[aslice[2]],
tile.latitudes[aslice[1]]
) for aslice in valid_indices])
matchup_points.extend(primary_points)
+ edge_results.extend(tile_to_edge_points(tile))
- # Convert tiles to 'edge points' which match the format of in-situ edge points.
- edge_results = []
- for matchup_tile in matchup_tiles:
- edge_results.extend(tile_to_edge_points(matchup_tile))
-
+ if len(matchup_points) <= 0:
+ return []
matchup_points = np.array(matchup_points)
print("%s Time to convert match points for partition %s to %s" % (
diff --git a/data-access/nexustiles/model/nexusmodel.py b/data-access/nexustiles/model/nexusmodel.py
index f5c9df6..7db4d61 100644
--- a/data-access/nexustiles/model/nexusmodel.py
+++ b/data-access/nexustiles/model/nexusmodel.py
@@ -17,7 +17,7 @@ from collections import namedtuple
import numpy as np
from dataclasses import dataclass
-
+from functools import reduce
NexusPoint = namedtuple('NexusPoint', 'latitude longitude depth time index data_vals')
BBox = namedtuple('BBox', 'min_lat max_lat min_lon max_lon')
@@ -156,12 +156,8 @@ class Tile(object):
if include_nan:
return list(np.ndindex(self.data.shape))
if self.is_multi:
- # For each variable, combine masks. This is a logical or
- # operation, because we want to ensure we don't lose any
- # data.
- combined_data_mask = np.logical_or(*self.data)
- # Return the indices where the data is valid
- return np.argwhere(combined_data_mask)
+ combined_data_inv_mask = reduce(np.logical_and, [data.mask for data in self.data])
+ return np.argwhere(np.logical_not(combined_data_inv_mask))
else:
return np.transpose(np.where(np.ma.getmaskarray(self.data) == False)).tolist()