You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by sk...@apache.org on 2023/09/25 22:50:31 UTC
[incubator-sdap-nexus] branch SDAP-493 created (now e0a5999)
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a change to branch SDAP-493
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
at e0a5999 update changelog
This branch includes the following new commits:
new b5df944 removed resultSizeLimit param from matchup
new 5e0fbb2 Add # of primaries/avergae secondaries to job output
new fbad6b7 rename to executionId
new e0a5999 update changelog
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-sdap-nexus] 01/04: removed resultSizeLimit param from matchup
Posted by sk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a commit to branch SDAP-493
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit b5df944ec6dcc3bb038c02f5849fd300c0a219c6
Author: skorper <st...@gmail.com>
AuthorDate: Mon Sep 25 13:37:09 2023 -0700
removed resultSizeLimit param from matchup
---
analysis/webservice/algorithms_spark/Matchup.py | 17 +++--------------
analysis/webservice/apidocs/openapi.yml | 13 -------------
2 files changed, 3 insertions(+), 27 deletions(-)
diff --git a/analysis/webservice/algorithms_spark/Matchup.py b/analysis/webservice/algorithms_spark/Matchup.py
index a55f61d..77ecc34 100644
--- a/analysis/webservice/algorithms_spark/Matchup.py
+++ b/analysis/webservice/algorithms_spark/Matchup.py
@@ -137,14 +137,6 @@ class Matchup(NexusCalcSparkTornadoHandler):
+ "If true, only the nearest point will be returned for each primary point. "
+ "If false, all points within the tolerances will be returned for each primary point. Default: False"
},
- "resultSizeLimit": {
- "name": "Result Size Limit",
- "type": "int",
- "description": "Optional integer value that limits the number of results returned from the matchup. "
- "If the number of primary matches is greater than this limit, the service will respond with "
- "(HTTP 202: Accepted) and an empty response body. A value of 0 means return all results. "
- "Default: 500"
- },
"prioritizeDistance": {
"name": "Prioritize distance",
"type": "boolean",
@@ -223,8 +215,6 @@ class Matchup(NexusCalcSparkTornadoHandler):
match_once = request.get_boolean_arg("matchOnce", default=False)
- result_size_limit = request.get_int_arg("resultSizeLimit", default=500)
-
start_seconds_from_epoch = int((start_time - EPOCH).total_seconds())
end_seconds_from_epoch = int((end_time - EPOCH).total_seconds())
@@ -234,7 +224,7 @@ class Matchup(NexusCalcSparkTornadoHandler):
return bounding_polygon, primary_ds_name, secondary_ds_names, parameter_s, \
start_time, start_seconds_from_epoch, end_time, end_seconds_from_epoch, \
depth_min, depth_max, time_tolerance, radius_tolerance, \
- platforms, match_once, result_size_limit, prioritize_distance
+ platforms, match_once, prioritize_distance
def get_job_pool(self, tile_ids):
if len(tile_ids) > LARGE_JOB_THRESHOLD:
@@ -244,7 +234,7 @@ class Matchup(NexusCalcSparkTornadoHandler):
def async_calc(self, execution_id, tile_ids, bounding_polygon, primary_ds_name,
secondary_ds_names, parameter_s, start_time, end_time, depth_min,
depth_max, time_tolerance, radius_tolerance, platforms, match_once,
- result_size_limit, start, prioritize_distance):
+ start, prioritize_distance):
# Call spark_matchup
self.log.debug("Calling Spark Driver")
@@ -310,7 +300,7 @@ class Matchup(NexusCalcSparkTornadoHandler):
bounding_polygon, primary_ds_name, secondary_ds_names, parameter_s, \
start_time, start_seconds_from_epoch, end_time, end_seconds_from_epoch, \
depth_min, depth_max, time_tolerance, radius_tolerance, \
- platforms, match_once, result_size_limit, prioritize_distance = self.parse_arguments(request)
+ platforms, match_once, prioritize_distance = self.parse_arguments(request)
args = {
"primary": primary_ds_name,
@@ -380,7 +370,6 @@ class Matchup(NexusCalcSparkTornadoHandler):
radius_tolerance=radius_tolerance,
platforms=platforms,
match_once=match_once,
- result_size_limit=result_size_limit,
start=start,
prioritize_distance=prioritize_distance
))
diff --git a/analysis/webservice/apidocs/openapi.yml b/analysis/webservice/apidocs/openapi.yml
index ea9b16b..dc6fdb4 100644
--- a/analysis/webservice/apidocs/openapi.yml
+++ b/analysis/webservice/apidocs/openapi.yml
@@ -154,19 +154,6 @@ paths:
type: boolean
default: false
example: false
- - in: query
- name: resultSizeLimit
- description: |
- Optional integer value that limits the number of results
- returned from the matchup. If the number of primary matches
- is greater than this limit, the service will respond with
- (HTTP 202 Accepted) and an empty response body. A value of
- 0 means return all results.
- required: false
- schema:
- type: integer
- default: 500
- example: 500
- in: query
name: prioritizeDistance
description: |
[incubator-sdap-nexus] 03/04: rename to executionId
Posted by sk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a commit to branch SDAP-493
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit fbad6b72bf649709cdb51152d49f65bf0a7c4cac
Author: skorper <st...@gmail.com>
AuthorDate: Mon Sep 25 15:48:01 2023 -0700
rename to executionId
---
analysis/webservice/apidocs/openapi.yml | 4 ++--
analysis/webservice/webmodel/NexusExecutionResults.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/analysis/webservice/apidocs/openapi.yml b/analysis/webservice/apidocs/openapi.yml
index dc6fdb4..f5c57a3 100644
--- a/analysis/webservice/apidocs/openapi.yml
+++ b/analysis/webservice/apidocs/openapi.yml
@@ -684,7 +684,7 @@ paths:
- in: query
name: id
description: |
- The job execution ID
+ The execution ID
required: true
schema:
type: string
@@ -702,7 +702,7 @@ paths:
- in: query
name: id
description: |
- The job execution ID
+ The execution ID
required: true
schema:
type: string
diff --git a/analysis/webservice/webmodel/NexusExecutionResults.py b/analysis/webservice/webmodel/NexusExecutionResults.py
index 7cf9abb..c80914d 100644
--- a/analysis/webservice/webmodel/NexusExecutionResults.py
+++ b/analysis/webservice/webmodel/NexusExecutionResults.py
@@ -40,7 +40,7 @@ def construct_job_status(job_state, created, updated, execution_id, params, host
'rel': 'self'
}],
'params': params,
- 'jobID': execution_id
+ 'executionID': execution_id
}
[incubator-sdap-nexus] 02/04: Add # of primaries/avergae secondaries to job output
Posted by sk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a commit to branch SDAP-493
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 5e0fbb2521cc8ce2fa33281707aeb28950384a6b
Author: skorper <st...@gmail.com>
AuthorDate: Mon Sep 25 15:45:41 2023 -0700
Add # of primaries/avergae secondaries to job output
---
analysis/webservice/algorithms/doms/ExecutionStatus.py | 12 +++++++++++-
analysis/webservice/algorithms/doms/ResultsStorage.py | 4 ++--
analysis/webservice/webmodel/NexusExecutionResults.py | 13 +++++++++++--
3 files changed, 24 insertions(+), 5 deletions(-)
diff --git a/analysis/webservice/algorithms/doms/ExecutionStatus.py b/analysis/webservice/algorithms/doms/ExecutionStatus.py
index 1bae455..2add7b1 100644
--- a/analysis/webservice/algorithms/doms/ExecutionStatus.py
+++ b/analysis/webservice/algorithms/doms/ExecutionStatus.py
@@ -53,6 +53,14 @@ class ExecutionStatusHandler(BaseDomsHandler.BaseDomsQueryCalcHandler):
code=404
)
+ # Get execution stats. This call will raise an exception if the
+ # execution is not done.
+ with ResultsRetrieval(self.config) as retrieval:
+ try:
+ execution_stats = retrieval.retrieveStats(execution_id)
+ except NexusProcessingException:
+ execution_stats = {}
+
job_status = NexusExecutionResults.ExecutionStatus(execution_details['status'])
host = f'{request.requestHandler.request.protocol}://{request.requestHandler.request.host}'
@@ -63,5 +71,7 @@ class ExecutionStatusHandler(BaseDomsHandler.BaseDomsQueryCalcHandler):
execution_id=execution_id,
message=execution_details['message'],
params=execution_params,
- host=host
+ host=host,
+ num_primary_matched=execution_stats.get('numPrimaryMatched'),
+ num_secondary_matched=execution_stats.get('numSecondaryMatched')
)
diff --git a/analysis/webservice/algorithms/doms/ResultsStorage.py b/analysis/webservice/algorithms/doms/ResultsStorage.py
index 39db27b..99e3c6b 100644
--- a/analysis/webservice/algorithms/doms/ResultsStorage.py
+++ b/analysis/webservice/algorithms/doms/ResultsStorage.py
@@ -286,7 +286,7 @@ class ResultsRetrieval(AbstractResultsContainer):
execution_id = uuid.UUID(execution_id)
params = self.retrieveParams(execution_id)
- stats = self.__retrieveStats(execution_id)
+ stats = self.retrieveStats(execution_id)
data = self.__retrieveData(execution_id, trim_data=trim_data, page_num=page_num, page_size=page_size)
return params, stats, data
@@ -357,7 +357,7 @@ class ResultsRetrieval(AbstractResultsContainer):
return entry
- def __retrieveStats(self, id):
+ def retrieveStats(self, id):
cql = "SELECT num_gridded_matched, num_insitu_matched, time_to_complete FROM doms_execution_stats where execution_id = %s limit 1"
rows = self._session.execute(cql, (id,))
for row in rows:
diff --git a/analysis/webservice/webmodel/NexusExecutionResults.py b/analysis/webservice/webmodel/NexusExecutionResults.py
index d5c1204..7cf9abb 100644
--- a/analysis/webservice/webmodel/NexusExecutionResults.py
+++ b/analysis/webservice/webmodel/NexusExecutionResults.py
@@ -44,7 +44,8 @@ def construct_job_status(job_state, created, updated, execution_id, params, host
}
-def construct_done(status, created, completed, execution_id, params, host):
+def construct_done(status, created, completed, execution_id, params, host,
+ num_primary_matched, num_secondary_matched):
job_body = construct_job_status(
status,
created,
@@ -53,6 +54,9 @@ def construct_done(status, created, completed, execution_id, params, host):
params,
host
)
+ # Add stats to body
+ job_body['totalPrimaryMatched'] = num_primary_matched
+ job_body['averageSecondaryMatched'] = round(num_secondary_matched/num_primary_matched)
# Construct urls
formats = [
@@ -112,7 +116,8 @@ def construct_cancelled(status, created, completed, execution_id, params, host):
class NexusExecutionResults:
def __init__(self, status=None, created=None, completed=None, execution_id=None, message='',
- params=None, host=None, status_code=200):
+ params=None, host=None, status_code=200, num_primary_matched=None,
+ num_secondary_matched=None):
self.status_code = status_code
self.status = status
self.created = created
@@ -121,6 +126,8 @@ class NexusExecutionResults:
self.message = message
self.execution_params = params
self.host = host
+ self.num_primary_matched = num_primary_matched
+ self.num_secondary_matched = num_secondary_matched
def toJson(self):
params = {
@@ -132,6 +139,8 @@ class NexusExecutionResults:
}
if self.status == ExecutionStatus.SUCCESS:
params['completed'] = self.completed
+ params['num_primary_matched'] = self.num_primary_matched
+ params['num_secondary_matched'] = self.num_secondary_matched
construct = construct_done
elif self.status == ExecutionStatus.RUNNING:
construct = construct_running
[incubator-sdap-nexus] 04/04: update changelog
Posted by sk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a commit to branch SDAP-493
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit e0a5999792b466b502c65d7d042d911a5214a4ba
Author: skorper <st...@gmail.com>
AuthorDate: Mon Sep 25 15:50:20 2023 -0700
update changelog
---
CHANGELOG.md | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1178918..b8ed55b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,9 +24,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- SDAP-482: Updated Saildrone in situ endpoint in config file
- SDAP-485: Improved behavior for retrying failed Cassandra inserts when saving matchup results.
- SDAP-487: Improved result fetch speed for large matchup results by tweaking `doms.doms_data` schema to support querying by primary value id.
+- SDAP-493:
+ - Updated /job endpoint to use `executionId` terminology for consistency with existing `/cdmsresults` endpoint
+ - Updated /job endpoint with details about number of primary and secondary tiles.
### Deprecated
### Removed
- SDAP-465: Removed `climatology` directory.
+- SDAP-493:
+ - Removed `resultSizeLimit` from /match_spark endpoint
### Fixed
- SDAP-474: Fixed bug in CSV attributes where secondary dataset would be rendered as comma separated characters
- SDAP-475: Bug fixes for `/timeSeriesSpark` and `/timeAvgMapSpark`