You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@sdap.apache.org by GitBox <gi...@apache.org> on 2020/06/26 22:28:02 UTC
[GitHub] [incubator-sdap-nexus] echyam commented on a change in pull request #104: Sdap 252
echyam commented on a change in pull request #104:
URL: https://github.com/apache/incubator-sdap-nexus/pull/104#discussion_r446427377
##########
File path: analysis/webservice/algorithms_spark/VarianceSpark.py
##########
@@ -23,15 +23,16 @@
from nexustiles.nexustiles import NexusTileService
from pytz import timezone
-from webservice.NexusHandler import nexus_handler, SparkHandler
+from webservice.NexusHandler import nexus_handler
+from webservice.algorithms_spark.NexusCalcSparkHandler import NexusCalcSkarkHandler
Review comment:
typo lines 27, 35
##########
File path: analysis/webservice/algorithms_spark/TimeAvgMapSpark.py
##########
@@ -21,15 +21,18 @@
import shapely.geometry
from nexustiles.nexustiles import NexusTileService
from pytz import timezone
-from webservice.NexusHandler import nexus_handler, SparkHandler
+from webservice.NexusHandler import nexus_handler
+from webservice.algorithms_spark.NexusCalcSparkHandler import NexusCalcSkarkHandler
Review comment:
typo lines 25, 33
##########
File path: analysis/webservice/algorithms_spark/Matchup.py
##########
@@ -33,7 +33,8 @@
from shapely.geometry import box
from shapely.geos import ReadingError
-from webservice.NexusHandler import SparkHandler, nexus_handler
+from webservice.NexusHandler import nexus_handler
+from webservice.algorithms_spark.NexusCalcSparkHandler import NexusCalcSkarkHandler
Review comment:
typo lines 37, 54, 135
##########
File path: analysis/webservice/algorithms_spark/HofMoellerSpark.py
##########
@@ -27,7 +27,8 @@
from matplotlib.ticker import FuncFormatter
from nexustiles.nexustiles import NexusTileService
from pytz import timezone
-from webservice.NexusHandler import SparkHandler, nexus_handler
+from webservice.NexusHandler import nexus_handler
+from webservice.algorithms_spark.NexusCalcSparkHandler import NexusCalcSkarkHandler
Review comment:
typo lines 31, 104, 331, 335, 340, 382, 386, 391
##########
File path: analysis/webservice/webmodel/NexusRequestObject.py
##########
@@ -0,0 +1,227 @@
+import logging
+import re
+from datetime import datetime
+from decimal import Decimal
+
+from pytz import UTC
+from shapely.geometry import Polygon
+from webservice.webmodel.RequestParameters import RequestParameters
+from webservice.webmodel.StatsComputeOptions import StatsComputeOptions
+
+
+class NexusRequestObject(StatsComputeOptions):
+ shortNamePattern = re.compile("^[a-zA-Z0-9_\-,\.]+$")
+ floatingPointPattern = re.compile('[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?')
+
+ def __init__(self, reqHandler):
+ self.__log = logging.getLogger(__name__)
+ if reqHandler is None:
+ raise Exception("Request handler cannot be null")
+ self.requestHandler = reqHandler
+ StatsComputeOptions.__init__(self)
+
+ def get_argument(self, name, default=None):
+ return self.requestHandler.get_argument(name, default=default)
+
+ def get_list_int_arg(self, name, default=None):
+ arg = self.get_argument(name, default=default)
+ return arg.split(',')
+
+ def __validate_is_shortname(self, v):
+ if v is None or len(v) == 0:
+ return False
+ return self.shortNamePattern.match(v) is not None
+
+ def __validate_is_number(self, v):
+ if v is None or (type(v) == str and len(v) == 0):
+ return False
+ elif type(v) == int or type(v) == float:
+ return True
+ else:
+ return self.floatingPointPattern.match(v) is not None
+
+ def get_float_arg(self, name, default=0.0):
+ arg = self.get_argument(name, default)
+ if self.__validate_is_number(arg):
+ return float(arg)
+ else:
+ return default
+
+ def get_decimal_arg(self, name, default=0.0):
+ arg = self.get_argument(name, default)
+ if self.__validate_is_number(arg):
+ return Decimal(arg)
+ else:
+ if default is None:
+ return None
+ return Decimal(default)
+
+ def get_int_arg(self, name, default=0):
+ arg = self.get_argument(name, default)
+ if self.__validate_is_number(arg):
+ return int(arg)
+ else:
+ return default
+
+ def get_boolean_arg(self, name, default=False):
+ arg = self.get_argument(name, "false" if not default else "true")
+ return arg is not None and arg in ['true', '1', 't', 'y', 'yes', 'True', 'T', 'Y',
+ 'Yes', True]
+
+ def get_datetime_arg(self, name, default=None):
+ time_str = self.get_argument(name, default=default)
+ if time_str == default:
+ return default
+ try:
+ dt = datetime.strptime(time_str, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=UTC)
+ except ValueError:
+ dt = datetime.utcfromtimestamp(int(time_str)).replace(tzinfo=UTC)
+ return dt
+
+ def get_apply_seasonal_cycle_filter(self, default=True):
+ return self.get_boolean_arg(RequestParameters.SEASONAL_CYCLE_FILTER, default=default)
+
+ def get_max_lat(self, default=Decimal(90)):
+ return self.get_decimal_arg("maxLat", default)
+
+ def get_min_lat(self, default=Decimal(-90)):
+ return self.get_decimal_arg("minLat", default)
+
+ def get_max_lon(self, default=Decimal(180)):
+ return self.get_decimal_arg("maxLon", default)
+
+ def get_min_lon(self, default=Decimal(-180)):
+ return self.get_decimal_arg("minLon", default)
+
+ # added to fit the simplified version of TimeAvgMapSpark parse_argumemt
+ def get_bounding_box(self):
+
+ b = self.get_argument("b", '')
+ if b:
+ min_lon, min_lat, max_lon, max_lat = [float(e) for e in b.split(",")]
+ else:
+ max_lat = self.get_argument("maxLat", 90)
+ max_lat = Decimal(max_lat) if self.__validate_is_number(max_lat) else 90
+
+ min_lat = self.get_argument("minLat", -90)
+ min_lat = Decimal(min_lat) if self.__validate_is_number(min_lat) else -90
+
+ max_lon = self.get_argument("maxLon", 180)
+ max_lon = Decimal(max_lon) if self.__validate_is_number(max_lon) else 180
+
+ min_lon = self.get_argument("minLon", -90)
+ min_lon = Decimal(min_lon) if self.__validate_is_number(min_lon) else -90
+
+ return min_lon, min_lat, max_lon, max_lat
+
+
+ def get_bounding_polygon(self):
+ west, south, east, north = [float(b) for b in self.get_argument("b").split(",")]
+ polygon = Polygon([(west, south), (east, south), (east, north), (west, north), (west, south)])
+ return polygon
+
+ def get_dataset(self):
+ ds = self.get_argument(RequestParameters.DATASET, None)
+ if ds is not None and not self.__validate_is_shortname(ds):
+ raise Exception("Invalid shortname")
+ else:
+ return ds.split(",")
+
+ def get_metadata_filter(self):
+ return self.requestHandler.get_arguments(RequestParameters.METADATA_FILTER)
+
+ def get_environment(self):
+ env = self.get_argument(RequestParameters.ENVIRONMENT, None)
+ if env is None and "Origin" in self.requestHandler.request.headers:
+ origin = self.requestHandler.request.headers["Origin"]
+ if origin == "http://localhost:63342":
+ env = "DEV"
+ if origin == "https://sealevel.uat.earthdata.nasa.gov":
+ env = "UAT"
+ elif origin == "https://sealevel.sit.earthdata.nasa.gov":
+ env = "SIT"
+ elif origin == "https://sealevel.earthdata.nasa.gov":
+ env = "PROD"
Review comment:
these URLs are specific to SLCP
##########
File path: analysis/webservice/webmodel/CustomEncoder.py
##########
@@ -0,0 +1,49 @@
+import json
+from datetime import datetime
+from decimal import Decimal
+
+import numpy as np
+
+
+class CustomEncoder(json.JSONEncoder):
+ def default(self, obj):
+ """If input object is an ndarray it will be converted into a dict
+ holding dtype, shape and the data, base64 encoded.
+ """
+ numpy_types = (
+ np.bool_,
+ # np.bytes_, -- python `bytes` class is not json serializable
+ # np.complex64, -- python `complex` class is not json serializable
+ # np.complex128, -- python `complex` class is not json serializable
+ # np.complex256, -- python `complex` class is not json serializable
+ # np.datetime64, -- python `datetime.datetime` class is not json serializable
Review comment:
datetimes are quite common in our data - I wonder if we can convert to string?
##########
File path: analysis/webservice/algorithms/NexusCalcHandler.py
##########
@@ -0,0 +1,70 @@
+import time
+import types
+
+from nexustiles.nexustiles import NexusTileService
+
+
+class NexusCalcHandler(object):
+ @classmethod
+ def validate(cls):
+ if "calc" not in cls.__dict__ or not type(cls.__dict__["calc"]) == types.FunctionType:
+ raise Exception("Method 'calc' has not been declared")
+
+ if "path" not in cls.__dict__:
+ raise Exception("Property 'path' has not been defined")
+
+ if "name" not in cls.__dict__:
+ raise Exception("Property 'name' has not been defined")
+
+ if "description" not in cls.__dict__:
+ raise Exception("Property 'description' has not been defined")
+
+ if "params" not in cls.__dict__:
+ raise Exception("Property 'params' has not been defined")
+
+ def __init__(self, skipCassandra=False, skipSolr=False):
+
+ self._tile_service = NexusTileService(skipCassandra, skipSolr)
+
+ def calc(self, computeOptions, **args):
+ raise Exception("calc() not yet implemented")
+
+ def _mergeDicts(self, x, y):
+ z = x.copy()
+ z.update(y)
+ return z
+
+ def _now(self):
+ millis = int(round(time.time() * 1000))
+ return millis
+
+ def _mergeDataSeries(self, resultsData, dataNum, resultsMap):
+
+ for entry in resultsData:
+
+ # frmtdTime = datetime.fromtimestamp(entry["time"] ).strftime("%Y-%m")
+ frmtdTime = entry["time"]
+
+ if not frmtdTime in resultsMap:
+ resultsMap[frmtdTime] = []
+ entry["ds"] = dataNum
+ resultsMap[frmtdTime].append(entry)
+
+ def _resultsMapToList(self, resultsMap):
+ resultsList = []
+ for key, value in resultsMap.iteritems():
+ resultsList.append(value)
+
+ resultsList = sorted(resultsList, key=lambda entry: entry[0]["time"])
+ return resultsList
Review comment:
I'm curious why we're using a 2-tuple here (maybe Joe can elaborate)
##########
File path: analysis/webservice/algorithms_spark/CorrMapSpark.py
##########
@@ -22,22 +22,17 @@
from nexustiles.nexustiles import NexusTileService
# from time import time
-from webservice.NexusHandler import nexus_handler, SparkHandler, DEFAULT_PARAMETERS_SPEC
+from webservice.NexusHandler import nexus_handler, DEFAULT_PARAMETERS_SPEC
+from webservice.algorithms_spark.NexusCalcSparkHandler import NexusCalcSkarkHandler
Review comment:
typo, lines 26, 31
##########
File path: analysis/webservice/algorithms_spark/DailyDifferenceAverageSpark.py
##########
@@ -23,7 +23,8 @@
from shapely import wkt
from shapely.geometry import Polygon
-from webservice.NexusHandler import nexus_handler, SparkHandler
+from webservice.NexusHandler import nexus_handler
+from webservice.algorithms_spark.NexusCalcSparkHandler import NexusCalcSkarkHandler
Review comment:
typo lines 27, 41, 75
##########
File path: analysis/webservice/algorithms_spark/ClimMapSpark.py
##########
@@ -22,22 +22,17 @@
import numpy as np
from nexustiles.nexustiles import NexusTileService
-from webservice.NexusHandler import nexus_handler, SparkHandler, DEFAULT_PARAMETERS_SPEC
+from webservice.NexusHandler import nexus_handler, DEFAULT_PARAMETERS_SPEC
+from webservice.algorithms_spark.NexusCalcSparkHandler import NexusCalcSkarkHandler
from webservice.webmodel import NexusResults, NexusProcessingException, NoDataException
@nexus_handler
-class ClimMapSparkHandlerImpl(SparkHandler):
+class ClimMapNexusSkarkHandlerImpl(NexusCalcSkarkHandler):
Review comment:
typo
##########
File path: analysis/webservice/algorithms_spark/ClimMapSpark.py
##########
@@ -66,13 +61,13 @@ def _map(tile_in_spark):
# print 'nexus call start at time %f' % t1
# sys.stdout.flush()
nexus_tiles = \
- ClimMapSparkHandlerImpl.query_by_parts(tile_service,
- min_lat, max_lat,
- min_lon, max_lon,
- ds,
- t_start,
- t_end,
- part_dim=2)
+ ClimMapNexusSkarkHandlerImpl.query_by_parts(tile_service,
Review comment:
typo
##########
File path: analysis/webservice/algorithms_spark/TimeSeriesSpark.py
##########
@@ -32,7 +32,8 @@
from pytz import timezone
from scipy import stats
from webservice import Filtering as filtering
-from webservice.NexusHandler import nexus_handler, SparkHandler
+from webservice.NexusHandler import nexus_handler
+from webservice.algorithms_spark.NexusCalcSparkHandler import NexusCalcSkarkHandler
Review comment:
typo lines 36, 46, 249
##########
File path: analysis/webservice/webapp.py
##########
@@ -203,22 +73,21 @@ def async_callback(self, result):
spark_context = None
for clazzWrapper in NexusHandler.AVAILABLE_HANDLERS:
- if issubclass(clazzWrapper.clazz(), NexusHandler.SparkHandler):
+ if issubclass(clazzWrapper, webservice.algorithms_spark.NexusCalcSparkHandler.NexusCalcSkarkHandler):
Review comment:
typo Skark
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org