[incubator-sdap-nexus] branch master updated: SDAP-438: Fix ResultStorage failure (#228)

[sk...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

skperez pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git


The following commit(s) were added to refs/heads/master by this push:
     new 5471b16  SDAP-438: Fix ResultStorage failure (#228)
5471b16 is described below

commit 5471b16b36abb6d6101fa5197eca9458e6a9be36
Author: Riley Kuttruff <72...@users.noreply.github.com>
AuthorDate: Thu Mar 23 09:32:33 2023 -0700

    SDAP-438: Fix ResultStorage failure (#228)
    
    * SDAP-438: Replace variable value NaN with None to fix error in result storage
    
    * Cleaned up error logging for batch prep
    
    ---------
    
    Co-authored-by: rileykk <ri...@jpl.nasa.gov>
---
 CHANGELOG.md                                       |  1 +
 .../webservice/algorithms/doms/ResultsStorage.py   | 53 ++++++++++++++--------
 2 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index dc3baeb..b03fda5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - SDAP-415: Fixed bug where mask was incorrectly combined across all variables for multi-variable satellite to satellite matchup
 - SDAP-434: Fix for webapp Docker image build failure
 - SDAP-412: Explicit definition of `__eq__` and `__hash__` in matchup `DomsPoint` class. This ensures all primary-secondary pairs with the same primary point are merged in the `combineByKey` step.
+- SDAP-438: Replace variable value NaN with None to fix error in result storage
 ### Security
 
 ## [1.0.0] - 2022-12-05
diff --git a/analysis/webservice/algorithms/doms/ResultsStorage.py b/analysis/webservice/algorithms/doms/ResultsStorage.py
index f0a18eb..c989286 100644
--- a/analysis/webservice/algorithms/doms/ResultsStorage.py
+++ b/analysis/webservice/algorithms/doms/ResultsStorage.py
@@ -13,20 +13,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-
 import configparser
+import json
 import logging
+import math
 import uuid
-import numpy as np
 from datetime import datetime
 
+import numpy as np
 import pkg_resources
+from cassandra.auth import PlainTextAuthProvider
 from cassandra.cluster import Cluster
 from cassandra.policies import TokenAwarePolicy, DCAwareRoundRobinPolicy
 from cassandra.query import BatchStatement
-from cassandra.auth import PlainTextAuthProvider
 from pytz import UTC
+from webservice.algorithms.doms.BaseDomsHandler import DomsEncoder
 
 
 class AbstractResultsContainer:
@@ -98,6 +99,7 @@ class ResultsStorage(AbstractResultsContainer):
         AbstractResultsContainer.__init__(self, config)
 
     def insertResults(self, results, params, stats, startTime, completeTime, userEmail, execution_id=None):
+        self._log.info('Beginning results write')
         if isinstance(execution_id, str):
             execution_id = uuid.UUID(execution_id)
 
@@ -105,6 +107,7 @@ class ResultsStorage(AbstractResultsContainer):
         self.__insertParams(execution_id, params)
         self.__insertStats(execution_id, stats)
         self.__insertResults(execution_id, results)
+        self._log.info('Results write finished')
         return execution_id
 
     def insertExecution(self, execution_id, startTime, completeTime, userEmail):
@@ -177,21 +180,29 @@ class ResultsStorage(AbstractResultsContainer):
 
         dataMap = self.__buildDataMap(data_dict)
         result_id = uuid.uuid4()
-        batch.add(insertStatement, (
-            result_id,
-            execution_id,
-            result["id"],
-            primaryId,
-            result["lon"],
-            result["lat"],
-            result["source"],
-            result["time"],
-            result["platform"] if "platform" in result else None,
-            result["device"] if "device" in result else None,
-            dataMap,
-            1 if primaryId is None else 0,
-            result["depth"]
-        ))
+
+        insert_params = (
+                result_id,
+                execution_id,
+                result["id"],
+                primaryId,
+                result["lon"],
+                result["lat"],
+                result["source"],
+                result["time"],
+                result["platform"] if "platform" in result else None,
+                result["device"] if "device" in result else None,
+                dataMap,
+                1 if primaryId is None else 0,
+                result["depth"]
+            )
+
+        try:
+            batch.add(insertStatement, insert_params)
+        except:
+            self._log.error(f'Result batch INSERT preparation failed')
+            self._log.error('INSERT params %s', str(insert_params))
+            raise
 
         n = 0
         if "matches" in result:
@@ -221,6 +232,10 @@ class ResultsStorage(AbstractResultsContainer):
             value = data_dict['variable_value']
             if isinstance(value, np.generic):
                 value = value.item()
+
+            if math.isnan(value):
+                value = None
+
             dataMap[name] = value
         return dataMap