[incubator-sdap-nexus] 01/02: fixed sat to sat matchup bug

[sk...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

skperez pushed a commit to branch feature/SDAP-415
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git

commit dbf61f23d45af63bf97d28a5b01ccf323edd4f39
Author: skorper <st...@gmail.com>
AuthorDate: Mon Mar 13 23:15:36 2023 -0700

    fixed sat to sat matchup bug
---
 CHANGELOG.md                                    |  1 +
 analysis/webservice/algorithms_spark/Matchup.py | 27 ++++++++++++++++---------
 data-access/nexustiles/model/nexusmodel.py      | 10 +++------
 3 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ae01564..c108fc5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Made `platforms` param optional in `/cdmssubset`, and removed int requirement
 - Updated OpenAPI specification for `/cdmssubset` to accurately reflect `platforms` and `parameter` field options.
 - SDAP-436: Added special case for handling Cassandra SwathMulti tiles with uniform time arrays
+- SDAP-415: Fixed bug where mask was incorrectly combined across all variables for multi-variable satellite to satellite matchup
 ### Security
 
 ## [1.0.0] - 2022-12-05
diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index 5c5ec74..2bc91c3 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -169,10 +169,11 @@ class Matchup(NexusCalcSparkHandler):
             raise NexusProcessingException(reason="'secondary' argument is required", code=400)
 
         parameter_s = request.get_argument('parameter')
-        insitu_params = get_insitu_params(insitu_schema.get())
-        if parameter_s and parameter_s not in insitu_params:
-            raise NexusProcessingException(
-                reason=f"Parameter {parameter_s} not supported. Must be one of {insitu_params}", code=400)
+        if parameter_s:
+            insitu_params = get_insitu_params(insitu_schema.get())
+            if parameter_s not in insitu_params:
+                raise NexusProcessingException(
+                    reason=f"Parameter {parameter_s} not supported. Must be one of {insitu_params}", code=400)
 
         try:
             start_time = request.get_start_datetime()
@@ -801,31 +802,37 @@ def match_satellite_to_insitu(tile_ids, primary_b, secondary_b, parameter_b, tt_
         polygon = Polygon(
             [(west, south), (east, south), (east, north), (west, north), (west, south)])
 
+        # Find tile IDS from spatial/temporal bounds of partition
         matchup_tiles = tile_service.find_tiles_in_polygon(
             bounding_polygon=polygon,
             ds=secondary_b.value,
             start_time=matchup_min_time,
             end_time=matchup_max_time,
-            fetch_data=True,
+            fl='id',
+            fetch_data=False,
             sort=['tile_min_time_dt asc', 'tile_min_lon asc', 'tile_min_lat asc'],
             rows=5000
         )
 
         # Convert Tile IDS to tiles and convert to UTM lat/lon projection.
         matchup_points = []
+        edge_results = []
         for tile in matchup_tiles:
+            # Retrieve tile data and convert to lat/lon projection
+            tiles = tile_service.find_tile_by_id(tile.tile_id, fetch_data=True)
+            tile = tiles[0]
+
             valid_indices = tile.get_indices()
+
             primary_points = np.array([aeqd_proj(
                 tile.longitudes[aslice[2]],
                 tile.latitudes[aslice[1]]
             ) for aslice in valid_indices])
             matchup_points.extend(primary_points)
+            edge_results.extend(tile_to_edge_points(tile))
 
-        # Convert tiles to 'edge points' which match the format of in-situ edge points.
-        edge_results = []
-        for matchup_tile in matchup_tiles:
-            edge_results.extend(tile_to_edge_points(matchup_tile))
-
+        if len(matchup_points) <= 0:
+            return []
         matchup_points = np.array(matchup_points)
 
     print("%s Time to convert match points for partition %s to %s" % (
diff --git a/data-access/nexustiles/model/nexusmodel.py b/data-access/nexustiles/model/nexusmodel.py
index f5c9df6..7db4d61 100644
--- a/data-access/nexustiles/model/nexusmodel.py
+++ b/data-access/nexustiles/model/nexusmodel.py
@@ -17,7 +17,7 @@ from collections import namedtuple
 
 import numpy as np
 from dataclasses import dataclass
-
+from functools import reduce
 
 NexusPoint = namedtuple('NexusPoint', 'latitude longitude depth time index data_vals')
 BBox = namedtuple('BBox', 'min_lat max_lat min_lon max_lon')
@@ -156,12 +156,8 @@ class Tile(object):
         if include_nan:
             return list(np.ndindex(self.data.shape))
         if self.is_multi:
-            # For each variable, combine masks. This is a logical or
-            # operation, because we want to ensure we don't lose any
-            # data.
-            combined_data_mask = np.logical_or(*self.data)
-            # Return the indices where the data is valid
-            return np.argwhere(combined_data_mask)
+            combined_data_inv_mask = reduce(np.logical_and, [data.mask for data in self.data])
+            return np.argwhere(np.logical_not(combined_data_inv_mask))
         else:
             return np.transpose(np.where(np.ma.getmaskarray(self.data) == False)).tolist()