You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by rk...@apache.org on 2023/08/22 19:03:10 UTC
[incubator-sdap-nexus] branch master updated: SDAP-475 - Similar fix for additional Spark algorithms (#272)
This is an automated email from the ASF dual-hosted git repository.
rkk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
The following commit(s) were added to refs/heads/master by this push:
new a02d537 SDAP-475 - Similar fix for additional Spark algorithms (#272)
a02d537 is described below
commit a02d5376eacbf442b9a1e30921657187ba59e150
Author: Riley Kuttruff <72...@users.noreply.github.com>
AuthorDate: Tue Aug 22 12:03:06 2023 -0700
SDAP-475 - Similar fix for additional Spark algorithms (#272)
* Bug fix for less than 12 months of climatology
* Bug fix for newer numpy versions breaking array creation when underlying sequence contains elements of different sizes
* Possible similar issue in HofMoellerSpark.py
* More algs w/ numpy issue
* Fix for VarianceSpark
---------
Co-authored-by: kevinmarlis <ke...@gmail.com>
Co-authored-by: rileykk <ri...@jpl.nasa.gov>
---
analysis/webservice/algorithms_spark/ClimMapSpark.py | 4 ++--
analysis/webservice/algorithms_spark/CorrMapSpark.py | 6 +++---
analysis/webservice/algorithms_spark/HofMoellerSpark.py | 2 +-
analysis/webservice/algorithms_spark/VarianceSpark.py | 6 +++---
4 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/analysis/webservice/algorithms_spark/ClimMapSpark.py b/analysis/webservice/algorithms_spark/ClimMapSpark.py
index 6bbacdb..8b34324 100644
--- a/analysis/webservice/algorithms_spark/ClimMapSpark.py
+++ b/analysis/webservice/algorithms_spark/ClimMapSpark.py
@@ -157,9 +157,9 @@ class ClimMapNexusSparkHandlerImpl(NexusCalcSparkHandler):
self._maxLonCent))
# Create array of tuples to pass to Spark map function
- nexus_tiles_spark = [[self._find_tile_bounds(t),
+ nexus_tiles_spark = np.array([[self._find_tile_bounds(t),
self._startTime, self._endTime,
- self._ds] for t in nexus_tiles]
+ self._ds] for t in nexus_tiles], dtype='object')
# print 'nexus_tiles_spark = ', nexus_tiles_spark
# Remove empty tiles (should have bounds set to None)
bad_tile_inds = np.where([t[0] is None for t in nexus_tiles_spark])[0]
diff --git a/analysis/webservice/algorithms_spark/CorrMapSpark.py b/analysis/webservice/algorithms_spark/CorrMapSpark.py
index fe1954d..71eaf72 100644
--- a/analysis/webservice/algorithms_spark/CorrMapSpark.py
+++ b/analysis/webservice/algorithms_spark/CorrMapSpark.py
@@ -57,7 +57,7 @@ class CorrMapNexusSparkHandlerImpl(NexusCalcSparkHandler):
# print 'days_at_a_time = ', days_at_a_time
t_incr = 86400 * days_at_a_time
- tile_service = tile_service_factory
+ tile_service = tile_service_factory()
# Compute the intermediate summations needed for the Pearson
# Correlation Coefficient. We use a one-pass online algorithm
@@ -205,9 +205,9 @@ class CorrMapNexusSparkHandlerImpl(NexusCalcSparkHandler):
self.log.debug('{0}, {1}'.format(i, datetime.utcfromtimestamp(d)))
# Create array of tuples to pass to Spark map function
- nexus_tiles_spark = [[self._find_tile_bounds(t),
+ nexus_tiles_spark = np.array([[self._find_tile_bounds(t),
self._startTime, self._endTime,
- self._ds] for t in nexus_tiles]
+ self._ds] for t in nexus_tiles], dtype='object')
# Remove empty tiles (should have bounds set to None)
bad_tile_inds = np.where([t[0] is None for t in nexus_tiles_spark])[0]
diff --git a/analysis/webservice/algorithms_spark/HofMoellerSpark.py b/analysis/webservice/algorithms_spark/HofMoellerSpark.py
index 6231bdb..d50006a 100644
--- a/analysis/webservice/algorithms_spark/HofMoellerSpark.py
+++ b/analysis/webservice/algorithms_spark/HofMoellerSpark.py
@@ -73,7 +73,7 @@ class HofMoellerCalculator(object):
for coord, points_at_coord in points_by_coord:
values_at_coord = np.array([[p.data_vals,
np.cos(np.radians(p.latitude))]
- for p in points_at_coord])
+ for p in points_at_coord], dtype='object')
vals = np.nan_to_num(values_at_coord[:, 0])
weights = values_at_coord[:, 1]
coord_cnt = len(values_at_coord)
diff --git a/analysis/webservice/algorithms_spark/VarianceSpark.py b/analysis/webservice/algorithms_spark/VarianceSpark.py
index 07922e6..7c217da 100644
--- a/analysis/webservice/algorithms_spark/VarianceSpark.py
+++ b/analysis/webservice/algorithms_spark/VarianceSpark.py
@@ -180,9 +180,9 @@ class VarianceNexusSparkHandlerImpl(NexusCalcSparkHandler):
self._maxLonCent))
# Create array of tuples to pass to Spark map function
- nexus_tiles_spark = [[self._find_tile_bounds(t),
- self._startTime, self._endTime,
- self._ds] for t in nexus_tiles]
+ nexus_tiles_spark = np.array([[self._find_tile_bounds(t),
+ self._startTime, self._endTime,
+ self._ds] for t in nexus_tiles], dtype='object')
# Remove empty tiles (should have bounds set to None)
bad_tile_inds = np.where([t[0] is None for t in nexus_tiles_spark])[0]