You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by sk...@apache.org on 2022/06/15 09:15:21 UTC
[incubator-sdap-nexus] branch matchup-bugfixes created (now 36bdff7)
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a change to branch matchup-bugfixes
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
at 36bdff7 Fix matchup bugs
This branch includes the following new commits:
new 36bdff7 Fix matchup bugs
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-sdap-nexus] 01/01: Fix matchup bugs
Posted by sk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a commit to branch matchup-bugfixes
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 36bdff74b246194fb1ae3779492779a252fb3e2c
Author: skorper <st...@gmail.com>
AuthorDate: Wed Jun 15 02:15:10 2022 -0700
Fix matchup bugs
---
.../webservice/algorithms/doms/DatasetListQuery.py | 25 ++++++++++++++--------
analysis/webservice/algorithms_spark/Matchup.py | 3 ++-
analysis/webservice/apidocs/openapi.yml | 25 +++++++++++-----------
3 files changed, 30 insertions(+), 23 deletions(-)
diff --git a/analysis/webservice/algorithms/doms/DatasetListQuery.py b/analysis/webservice/algorithms/doms/DatasetListQuery.py
index 0d2e77f..ed0ca7e 100644
--- a/analysis/webservice/algorithms/doms/DatasetListQuery.py
+++ b/analysis/webservice/algorithms/doms/DatasetListQuery.py
@@ -98,15 +98,22 @@ class DomsDatasetListQueryHandler(BaseDomsHandler.BaseDomsQueryCalcHandler):
for satellite in satellitesList:
satellite["metadata"] = self.getMetadataForSource(satellite["shortName"])
- for insitu in config.ENDPOINTS:
- depths, facets = self.getFacetsForInsituSource(insitu)
- insituList.append({
- "name": insitu["name"],
- "endpoint": insitu["url"],
- "metadata": self.getMetadataForSource(insitu["name"]),
- "depths": depths,
- "facets": facets
- })
+ for insitu in config.INSITU_PROVIDER_MAP:
+ provider = insitu['name']
+ for project in insitu['projects']:
+ project_name = project['name']
+ insituList.append({
+ 'name': project_name,
+ 'endpoint': f'{config.INSITU_API_ENDPOINT}?provider={provider}&project={project_name}',
+ 'metadata': {
+ 'Projects': [
+ {
+ 'ShortName': project_name,
+ 'LongName': project_name
+ }
+ ]
+ }
+ })
values = {
"satellite": satellitesList,
diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index cf20c5f..0f7e5ee 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -854,13 +854,14 @@ def query_edge(dataset, variable, startTime, endTime, bbox, platform, depth_min,
# Get all edge results
next_page_url = edge_endpoints.getEndpoint()
while next_page_url is not None and next_page_url != 'NA':
- logging.debug(f'Edge request {next_page_url}')
+ the_time = datetime.now()
if session is not None:
edge_page_request = session.get(next_page_url, params=params)
else:
edge_page_request = requests.get(next_page_url, params=params)
edge_page_request.raise_for_status()
+ logging.info("%s Time to query edge %s", datetime.now() - the_time, edge_page_request.url)
edge_page_response = json.loads(edge_page_request.text)
diff --git a/analysis/webservice/apidocs/openapi.yml b/analysis/webservice/apidocs/openapi.yml
index 2754463..ffe754e 100644
--- a/analysis/webservice/apidocs/openapi.yml
+++ b/analysis/webservice/apidocs/openapi.yml
@@ -14,10 +14,11 @@ tags:
- name: Subsetting
description: Data Subsetting API
paths:
- /match_spark_doms:
+ /match_spark:
get:
- summary: Execute matchup request
+ summary: Execute matchup request (AWS insitu)
operationId: matchup
+ description: Endpoint to execute matchup request. If secondary dataset is insitu, use AWS insitu API.
tags:
- Matchup
parameters:
@@ -30,7 +31,7 @@ paths:
schema:
type: string
x-dspopulate:
- - satellite
+ - satellite
example: avhrr-l4-glob-v2-daily-ncei-ghrsst-sstblend-avhrr-oi-glob-v020-fv020
- in: query
name: secondary
@@ -43,8 +44,8 @@ paths:
schema:
type: string
x-dspopulate:
- - satellite
- - insitu
+ - satellite
+ - insitu
example: icoads
- in: query
name: startTime
@@ -126,7 +127,6 @@ paths:
required: false
schema:
type: string
- enum: ['sst', 'sss', 'wind']
default: sst
example: sst
- in: query
@@ -174,10 +174,11 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/Error'
- /match_spark:
+ /match_spark_doms:
get:
- summary: Execute matchup request
- operationId: matchup
+ summary: Execute matchup request (DOMS insitu)
+ operationId: matchup_doms
+ description: Endpoint to execute matchup request. If secondary dataset is insitu, use DOMS insitu API.
tags:
- Matchup
parameters:
@@ -190,7 +191,7 @@ paths:
schema:
type: string
x-dspopulate:
- - satellite
+ - satellite
example: avhrr-l4-glob-v2-daily-ncei-ghrsst-sstblend-avhrr-oi-glob-v020-fv020
- in: query
name: secondary
@@ -202,9 +203,6 @@ paths:
required: true
schema:
type: string
- x-dspopulate:
- - satellite
- - insitu
example: icoads
- in: query
name: startTime
@@ -286,6 +284,7 @@ paths:
required: false
schema:
type: string
+ enum: ['sst', 'sss', 'wind']
default: sst
example: sst
- in: query