You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by sk...@apache.org on 2022/06/15 09:15:22 UTC
[incubator-sdap-nexus] 01/01: Fix matchup bugs
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a commit to branch matchup-bugfixes
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 36bdff74b246194fb1ae3779492779a252fb3e2c
Author: skorper <st...@gmail.com>
AuthorDate: Wed Jun 15 02:15:10 2022 -0700
Fix matchup bugs
---
.../webservice/algorithms/doms/DatasetListQuery.py | 25 ++++++++++++++--------
analysis/webservice/algorithms_spark/Matchup.py | 3 ++-
analysis/webservice/apidocs/openapi.yml | 25 +++++++++++-----------
3 files changed, 30 insertions(+), 23 deletions(-)
diff --git a/analysis/webservice/algorithms/doms/DatasetListQuery.py b/analysis/webservice/algorithms/doms/DatasetListQuery.py
index 0d2e77f..ed0ca7e 100644
--- a/analysis/webservice/algorithms/doms/DatasetListQuery.py
+++ b/analysis/webservice/algorithms/doms/DatasetListQuery.py
@@ -98,15 +98,22 @@ class DomsDatasetListQueryHandler(BaseDomsHandler.BaseDomsQueryCalcHandler):
for satellite in satellitesList:
satellite["metadata"] = self.getMetadataForSource(satellite["shortName"])
- for insitu in config.ENDPOINTS:
- depths, facets = self.getFacetsForInsituSource(insitu)
- insituList.append({
- "name": insitu["name"],
- "endpoint": insitu["url"],
- "metadata": self.getMetadataForSource(insitu["name"]),
- "depths": depths,
- "facets": facets
- })
+ for insitu in config.INSITU_PROVIDER_MAP:
+ provider = insitu['name']
+ for project in insitu['projects']:
+ project_name = project['name']
+ insituList.append({
+ 'name': project_name,
+ 'endpoint': f'{config.INSITU_API_ENDPOINT}?provider={provider}&project={project_name}',
+ 'metadata': {
+ 'Projects': [
+ {
+ 'ShortName': project_name,
+ 'LongName': project_name
+ }
+ ]
+ }
+ })
values = {
"satellite": satellitesList,
diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index cf20c5f..0f7e5ee 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -854,13 +854,14 @@ def query_edge(dataset, variable, startTime, endTime, bbox, platform, depth_min,
# Get all edge results
next_page_url = edge_endpoints.getEndpoint()
while next_page_url is not None and next_page_url != 'NA':
- logging.debug(f'Edge request {next_page_url}')
+ the_time = datetime.now()
if session is not None:
edge_page_request = session.get(next_page_url, params=params)
else:
edge_page_request = requests.get(next_page_url, params=params)
edge_page_request.raise_for_status()
+ logging.info("%s Time to query edge %s", datetime.now() - the_time, edge_page_request.url)
edge_page_response = json.loads(edge_page_request.text)
diff --git a/analysis/webservice/apidocs/openapi.yml b/analysis/webservice/apidocs/openapi.yml
index 2754463..ffe754e 100644
--- a/analysis/webservice/apidocs/openapi.yml
+++ b/analysis/webservice/apidocs/openapi.yml
@@ -14,10 +14,11 @@ tags:
- name: Subsetting
description: Data Subsetting API
paths:
- /match_spark_doms:
+ /match_spark:
get:
- summary: Execute matchup request
+ summary: Execute matchup request (AWS insitu)
operationId: matchup
+ description: Endpoint to execute matchup request. If secondary dataset is insitu, use AWS insitu API.
tags:
- Matchup
parameters:
@@ -30,7 +31,7 @@ paths:
schema:
type: string
x-dspopulate:
- - satellite
+ - satellite
example: avhrr-l4-glob-v2-daily-ncei-ghrsst-sstblend-avhrr-oi-glob-v020-fv020
- in: query
name: secondary
@@ -43,8 +44,8 @@ paths:
schema:
type: string
x-dspopulate:
- - satellite
- - insitu
+ - satellite
+ - insitu
example: icoads
- in: query
name: startTime
@@ -126,7 +127,6 @@ paths:
required: false
schema:
type: string
- enum: ['sst', 'sss', 'wind']
default: sst
example: sst
- in: query
@@ -174,10 +174,11 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/Error'
- /match_spark:
+ /match_spark_doms:
get:
- summary: Execute matchup request
- operationId: matchup
+ summary: Execute matchup request (DOMS insitu)
+ operationId: matchup_doms
+ description: Endpoint to execute matchup request. If secondary dataset is insitu, use DOMS insitu API.
tags:
- Matchup
parameters:
@@ -190,7 +191,7 @@ paths:
schema:
type: string
x-dspopulate:
- - satellite
+ - satellite
example: avhrr-l4-glob-v2-daily-ncei-ghrsst-sstblend-avhrr-oi-glob-v020-fv020
- in: query
name: secondary
@@ -202,9 +203,6 @@ paths:
required: true
schema:
type: string
- x-dspopulate:
- - satellite
- - insitu
example: icoads
- in: query
name: startTime
@@ -286,6 +284,7 @@ paths:
required: false
schema:
type: string
+ enum: ['sst', 'sss', 'wind']
default: sst
example: sst
- in: query