You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by sk...@apache.org on 2022/08/22 15:33:01 UTC
[incubator-sdap-nexus] 04/06: Updated matchup output fields. Validate matchup 'parameter' param against insitu schema
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a commit to branch proxy-bugfix-apidocs
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 2bab8cb23b23487848e24c310834e8e96be25d98
Author: skorper <st...@gmail.com>
AuthorDate: Mon Aug 22 08:22:58 2022 -0700
Updated matchup output fields. Validate matchup 'parameter' param against insitu schema
---
CHANGELOG.md | 4 ++-
analysis/webservice/algorithms_spark/Matchup.py | 39 ++++++++++++++-----------
2 files changed, 25 insertions(+), 18 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index aa47d27..3d97eef 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,13 +14,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- SDAP-371: Renamed `/domssubset` endpoint to `/cdmssubset`
- SDAP-390: Updated NetCDF reader tool for data matchup and added user functionality.
- SDAP-396: Added saildrone insitu api to matchup
+- Matchup validates insitu parameter using insitu API schema endpoint
### Changed
-
-SDAP-390: Changed `/doms` to `/cdms` and `doms_reader.py` to `cdms_reader.py`
- domslist endpoint points to AWS insitu instead of doms insitu
+- Matchup returns numSecondary and numPrimary counts rather than insitu/gridded
### Deprecated
### Removed
- removed dropdown from matchup doms endpoint secondary param
+- Matchup no longer returns insituMatched and griddedMatched fields
### Fixed
- Fix failing test_matchup unit test
- Fixed bug in OpenAPI spec where both matchup endpoints shared the same id
diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index b85c7b4..5dafb31 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -15,7 +15,6 @@
from typing import Optional
-import json
import logging
import threading
from shapely.geometry import Polygon
@@ -30,9 +29,7 @@ import requests
from pytz import timezone, UTC
from scipy import spatial
from shapely import wkt
-from shapely.geometry import Point
from shapely.geometry import box
-from shapely.geos import WKTReadingError
from webservice.NexusHandler import nexus_handler
from webservice.algorithms_spark.NexusCalcSparkHandler import NexusCalcSparkHandler
@@ -46,6 +43,7 @@ from webservice.webmodel import NexusProcessingException
EPOCH = timezone('UTC').localize(datetime(1970, 1, 1))
ISO_8601 = '%Y-%m-%dT%H:%M:%S%z'
+insitu_schema = query_insitu_schema()
def iso_time_to_epoch(str_time):
@@ -73,7 +71,7 @@ class Matchup(NexusCalcSparkHandler):
"parameter": {
"name": "Match-Up Parameter",
"type": "string",
- "description": "The parameter of interest used for the match up. One of 'sst', 'sss', 'wind'. Optional"
+ "description": "The parameter of interest used for the match up. Only used for satellite to insitu matchups. Optional"
},
"startTime": {
"name": "Start Time",
@@ -157,9 +155,10 @@ class Matchup(NexusCalcSparkHandler):
raise NexusProcessingException(reason="'secondary' argument is required", code=400)
parameter_s = request.get_argument('parameter')
- if parameter_s and parameter_s not in ['sst', 'sss', 'wind']:
+ insitu_params = get_insitu_params(insitu_schema)
+ if parameter_s and parameter_s not in insitu_params:
raise NexusProcessingException(
- reason="Parameter %s not supported. Must be one of 'sst', 'sss', 'wind'." % parameter_s, code=400)
+ reason=f"Parameter {parameter_s} not supported. Must be one of {insitu_params}", code=400)
try:
start_time = request.get_start_datetime()
@@ -262,10 +261,8 @@ class Matchup(NexusCalcSparkHandler):
total_values = sum(len(v) for v in spark_result.values())
details = {
"timeToComplete": int((end - start).total_seconds()),
- "numInSituRecords": 0,
- "numInSituMatched": total_values,
- "numGriddedChecked": 0,
- "numGriddedMatched": total_keys
+ "numSecondaryMatched": total_values,
+ "numPrimaryMatched": total_keys
}
matches = Matchup.convert_to_matches(spark_result)
@@ -432,13 +429,8 @@ class DomsPoint(object):
point.device = DomsPoint._variables_to_device(tile.variables)
return point
- insitu_schema = None
-
@staticmethod
def from_edge_point(edge_point):
- if DomsPoint.insitu_schema is None:
- DomsPoint.insitu_schema = query_insitu_schema()
-
point = DomsPoint()
x, y = edge_point['longitude'], edge_point['latitude']
@@ -513,7 +505,7 @@ class DomsPoint(object):
val = edge_point.get(name)
if not val:
continue
- unit = get_insitu_unit(name, DomsPoint.insitu_schema)
+ unit = get_insitu_unit(name, insitu_schema)
data.append(DataPoint(
variable_name=name,
cf_variable_name=name,
@@ -649,9 +641,22 @@ def add_meters_to_lon_lat(lon, lat, meters):
return longitude, latitude
+def get_insitu_params(insitu_schema):
+ """
+ Get all possible insitu params from the CDMS insitu schema
+ """
+ params = insitu_schema.get(
+ 'definitions', {}).get('observation', {}).get('properties', {})
+
+ # Filter params so only variables with units are considered
+ params = list(map(
+ lambda param: param[0], filter(lambda param: 'units'in param[1], params.items())))
+ return params
+
+
def get_insitu_unit(variable_name, insitu_schema):
"""
- Query the insitu API and retrieve the units for the given variable.
+ Retrieve the units from the insitu api schema endpoint for the given variable.
If no units are available for this variable, return "None"
"""
properties = insitu_schema.get('definitions', {}).get('observation', {}).get('properties', {})