You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by sk...@apache.org on 2023/12/20 16:31:20 UTC
(incubator-sdap-nexus) branch apidocs-improvements created (now 9ca1d2e)
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a change to branch apidocs-improvements
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
at 9ca1d2e updated openapi.yml
This branch includes the following new commits:
new 1e8cc4e Update helm cassandra dependency (#289)
new 9ca1d2e updated openapi.yml
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-sdap-nexus) 02/02: updated openapi.yml
Posted by sk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a commit to branch apidocs-improvements
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 9ca1d2e9386e627eb6010865fb7471c17ff41f43
Author: skorper <st...@gmail.com>
AuthorDate: Wed Dec 20 08:31:09 2023 -0800
updated openapi.yml
---
analysis/webservice/apidocs/openapi.yml | 329 ++++----------------------------
1 file changed, 36 insertions(+), 293 deletions(-)
diff --git a/analysis/webservice/apidocs/openapi.yml b/analysis/webservice/apidocs/openapi.yml
index 8c6efdc..a89701e 100644
--- a/analysis/webservice/apidocs/openapi.yml
+++ b/analysis/webservice/apidocs/openapi.yml
@@ -15,7 +15,9 @@
openapi: 3.0.3
info:
- description: The next generation cloud-based science data service platform.
+ description: |-
+ The next generation cloud-based science data service platform. </p>
+ <b><a href="mailto:cdms@jpl.nasa.gov">Click here to provide feedback</a></b>
version: 1.1.0
title: Science Data Analytics Platform (SDAP)
license:
@@ -24,8 +26,6 @@ info:
tags:
- name: Matchup
description: Data Matchup API
- - name: Analytics
- description: Data Analytics API
- name: Subsetting
description: Data Subsetting API
paths:
@@ -66,12 +66,14 @@ paths:
name: startTime
description: |
Starting time in format YYYY-MM-DDTHH:mm:ssZ or seconds
- since epoch
+ since epoch. Please be aware that limited data has been
+ ingested into the current system. Most satellite datasets
+ have data between 2017 and 2020.
required: true
schema:
type: string
format: date-time
- example: '2012-09-25T00:00:00Z'
+ example: '2018-09-25T00:00:00Z'
- in: query
name: endTime
description: |
@@ -81,7 +83,7 @@ paths:
schema:
type: string
format: date-time
- example: '2012-09-30T23:59:59Z'
+ example: '2018-09-30T23:59:59Z'
- in: query
name: b
description: |
@@ -151,7 +153,7 @@ paths:
required: false
schema:
type: boolean
- default: false
+ default: true
example: false
- in: query
name: prioritizeDistance
@@ -192,164 +194,49 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/Error'
- /match_spark_doms:
+ /job:
get:
- summary: Execute matchup request (DOMS insitu)
- operationId: matchup_doms
- description: Endpoint to execute matchup request. If secondary dataset is insitu, use DOMS insitu API.
+ summary: |
+ Get job status
+ operationId: job
tags:
- Matchup
+ description: "Get job status by execution id"
parameters:
- in: query
- name: primary
- description: |
- The primary dataset used to find matches for. One of the
- satellite "shortName" as supplied by /domslist endpoint.
- required: true
- schema:
- type: string
- x-dspopulate:
- - satellite
- example: avhrr-l4-glob-v2-daily-ncei-ghrsst-sstblend-avhrr-oi-glob-v020-fv020
- - in: query
- name: secondary
- description: |
- The dataset(s) being searched for measurements that match
- the measurements in primary. One or more (comma-separated)
- of the insitu or satellite "name" as supplied by
- https://doms.jpl.nasa.gov/domslist
- required: true
- schema:
- type: string
- example: icoads
- - in: query
- name: startTime
+ name: id
description: |
- Starting time in format YYYY-MM-DDTHH:mm:ssZ or seconds
- since epoch
+ The execution ID
required: true
schema:
type: string
- format: date-time
- example: '2012-09-25T00:00:00Z'
+ format: uuid
+ example: c864a51b-3d87-4872-9070-632820b1cae2
- in: query
- name: endTime
+ name: filename
description: |
- Ending time in format YYYY-MM-DDTHH:mm:ssZ or seconds
- since epoch
- required: true
+ Optional filename. Will be passed into /job results links
+ required: false
schema:
type: string
- format: date-time
- example: '2012-09-30T23:59:59Z'
+ /job/cancel:
+ get:
+ summary: |
+ Cancel running job
+ operationId: jobCancel
+ tags:
+ - Matchup
+ description: "Cancel running job"
+ parameters:
- in: query
- name: b
+ name: id
description: |
- Minimum (Western) Longitude, Minimum (Southern) Latitude,
- Maximum (Eastern) Longitude, Maximum (Northern) Latitude
- required: true
- schema:
- type: string
- example: -45,15,-30,30
- - in: query
- name: platforms
- description: Platforms to include for matchup consideration
+ The execution ID
required: true
schema:
type: string
- example: 1,2,3,4,5,6,7,8,9
- - in: query
- name: depthMin
- description: |
- Minimum depth of measurements allowed to be considered for
- matchup
- required: false
- schema:
- type: integer
- example: 0
- - in: query
- name: depthMax
- description: |
- Maximum depth of measurements allowed to be considered for
- matchup
- required: false
- schema:
- type: integer
- example: 5
- - in: query
- name: tt
- description: |
- Tolerance in time (seconds) when comparing two measurements.
- required: false
- schema:
- type: integer
- default: 86400
- example: 86400
- - in: query
- name: rt
- description: |
- Tolerance in radius (meters) when comparing two
- measurements.
- required: false
- schema:
- type: number
- default: 1000.0
- example: 1000.0
- - in: query
- name: parameter
- description: |
- The parameter of interest used for the match up.
- required: false
- schema:
- type: string
- enum: ['sst', 'sss', 'wind']
- default: sst
- example: sst
- - in: query
- name: matchOnce
- description: |
- True/False flag used to determine if more than one match
- per primary point is returned. If true, only the nearest
- point will be returned for each primary point. If false,
- all points within the tolerances will be returned for each
- primary point.
- required: false
- schema:
- type: boolean
- default: false
- example: false
- - in: query
- name: resultSizeLimit
- description: |
- Optional integer value that limits the number of results
- returned from the matchup. If the number of primary matches
- is greater than this limit, the service will respond with
- (HTTP 202 Accepted) and an empty response body. A value of
- 0 means return all results.
- required: false
- schema:
- type: integer
- default: 500
- example: 500
- responses:
- '200':
- description: Successful operation
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/MatchupResponse'
- '400':
- description: Bad request
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/Error'
- '500':
- description: Server error
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/Error'
+ format: uuid
+ example: c864a51b-3d87-4872-9070-632820b1cae2
/cdmssubset:
get:
summary: Subset CDMS sources given the search domain
@@ -388,7 +275,7 @@ paths:
schema:
type: string
format: date-time
- example: '2013-10-21T00:00:00Z'
+ example: '2018-10-21T00:00:00Z'
- in: query
name: endTime
description: |
@@ -398,7 +285,7 @@ paths:
schema:
type: string
format: date-time
- example: '2013-10-21T01:00:00Z'
+ example: '2018-10-21T01:00:00Z'
- in: query
name: b
description: |
@@ -577,150 +464,6 @@ paths:
application/json:
schema:
$ref: '#/components/schemas/Error'
- /stats:
- get:
- summary: |
- Computes time series statistics
- operationId: stats
- tags:
- - Analytics
- description: "NOTE: Only supports satellite datasets."
- parameters:
- - in: query
- name: ds
- description: |
- The dataset on which to generate the statistics
- required: true
- schema:
- type: string
- x-dspopulate:
- - satellite
- - insitu
- example: MUR25-JPL-L4-GLOB-v04.2
- - in: query
- name: minLat
- description: |
- Minimum latitude on which to generate statistics
- required: true
- schema:
- type: string
- example: -90
- - in: query
- name: maxLat
- description: |
- Maximum latitude on which to generate statistics
- required: true
- schema:
- type: string
- example: 90
- - in: query
- name: minLon
- description: |
- Minimum longitude on which to generate statistics
- required: true
- schema:
- type: string
- example: -180
- - in: query
- name: maxLon
- description: |
- Maximum longitude on which to generate statistics
- required: true
- schema:
- type: string
- example: 180
- - in: query
- name: startTime
- description: |
- Starting time in format YYYY-MM-DDTHH:mm:ssZ or seconds
- since epoch
- required: true
- schema:
- type: string
- format: date-time
- example: '2013-10-21T00:00:00Z'
- - in: query
- name: endTime
- description: |
- Ending time in format YYYY-MM-DDTHH:mm:ssZ or seconds
- since epoch
- required: true
- schema:
- type: string
- format: date-time
- example: '2013-10-31T23:59:59Z'
- - in: query
- name: output
- description: |
- Data formatting for results. Default is ‘JSON’. Alternative
- is ‘CSV’ for comma-seperated values.
- required: false
- schema:
- type: string
- enum: ['JSON', 'CSV']
- example: JSON
- responses:
- '200':
- description: Successful operation
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/Stats'
- '400':
- description: Bad request
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/Error'
- '500':
- description: Server error
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/Error'
- /job:
- get:
- summary: |
- Get job status
- operationId: job
- tags:
- - Analytics
- description: "Get job status by execution id"
- parameters:
- - in: query
- name: id
- description: |
- The execution ID
- required: true
- schema:
- type: string
- format: uuid
- example: c864a51b-3d87-4872-9070-632820b1cae2
- - in: query
- name: filename
- description: |
- Optional filename. Will be passed into /job results links
- required: false
- schema:
- type: string
- /job/cancel:
- get:
- summary: |
- Cancel running job
- operationId: jobCancel
- tags:
- - Analytics
- description: "Cancel running job"
- parameters:
- - in: query
- name: id
- description: |
- The execution ID
- required: true
- schema:
- type: string
- format: uuid
- example: c864a51b-3d87-4872-9070-632820b1cae2
externalDocs:
description: Documentation
url: https://incubator-sdap-nexus.readthedocs.io/en/latest/index.html
(incubator-sdap-nexus) 01/02: Update helm cassandra dependency (#289)
Posted by sk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
skperez pushed a commit to branch apidocs-improvements
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 1e8cc4e9d31d295e172c0db4bba61a5776642bea
Author: Riley Kuttruff <72...@users.noreply.github.com>
AuthorDate: Mon Nov 27 15:44:38 2023 -0800
Update helm cassandra dependency (#289)
* Update helm cassandra dependency
* Bump default cassandra PV to 4
* Bump default cassandra PV to 4 in tools
* Changelog
* Fixed small documentation issue
---------
Co-authored-by: rileykk <ri...@jpl.nasa.gov>
---
CHANGELOG.md | 1 +
analysis/webservice/algorithms/doms/ResultsStorage.py | 3 +++
analysis/webservice/algorithms/doms/domsconfig.ini.default | 2 +-
data-access/nexustiles/config/datastores.ini.default | 2 +-
helm/requirements.yaml | 2 +-
helm/values.yaml | 3 +--
tools/deletebyquery/deletebyquery.py | 2 +-
tools/doms-data-tools/update_doms_data_pk.py | 2 +-
tools/domspurge/README.md | 2 +-
tools/domspurge/purge.py | 2 +-
10 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6ffff5d..5e36c0a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -41,6 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Status code for results endpoint if execution id is not found fixed to be `404` instead of `500`.
- Ensured links in the `/job` endpoint are https
- SDAP-488: Workaround to build issue on Apple Silicon (M1/M2). Image build installs nexusproto through PyPI instead of building from source. A build arg `BUILD_NEXUSPROTO` was defined to allow building from source if desired
+- SDAP-496: Fix `solr-cloud-init` image failing to run.
### Security
## [1.1.0] - 2023-04-26
diff --git a/analysis/webservice/algorithms/doms/ResultsStorage.py b/analysis/webservice/algorithms/doms/ResultsStorage.py
index 48b2122..1dea161 100644
--- a/analysis/webservice/algorithms/doms/ResultsStorage.py
+++ b/analysis/webservice/algorithms/doms/ResultsStorage.py
@@ -65,6 +65,9 @@ class AbstractResultsContainer:
dc_policy = DCAwareRoundRobinPolicy(cassDatacenter)
token_policy = TokenAwarePolicy(dc_policy)
+ logger.info(f'Connecting to Cassandra cluster @ {[host for host in cassHost.split(",")]}; datacenter: '
+ f'{cassDatacenter}; protocol version: {cassVersion}')
+
self._cluster = Cluster([host for host in cassHost.split(',')], load_balancing_policy=token_policy,
protocol_version=cassVersion, auth_provider=auth_provider)
diff --git a/analysis/webservice/algorithms/doms/domsconfig.ini.default b/analysis/webservice/algorithms/doms/domsconfig.ini.default
index 55f9b16..f4e4496 100644
--- a/analysis/webservice/algorithms/doms/domsconfig.ini.default
+++ b/analysis/webservice/algorithms/doms/domsconfig.ini.default
@@ -18,7 +18,7 @@ host=localhost
port=9042
keyspace=doms
local_datacenter=datacenter1
-protocol_version=3
+protocol_version=4
dc_policy=DCAwareRoundRobinPolicy
username=
password=
diff --git a/data-access/nexustiles/config/datastores.ini.default b/data-access/nexustiles/config/datastores.ini.default
index d8db190..51455a3 100644
--- a/data-access/nexustiles/config/datastores.ini.default
+++ b/data-access/nexustiles/config/datastores.ini.default
@@ -18,7 +18,7 @@ host=localhost
port=9042
keyspace=nexustiles
local_datacenter=datacenter1
-protocol_version=3
+protocol_version=4
dc_policy=DCAwareRoundRobinPolicy
username=
password=
diff --git a/helm/requirements.yaml b/helm/requirements.yaml
index a999658..1de8cf0 100644
--- a/helm/requirements.yaml
+++ b/helm/requirements.yaml
@@ -12,7 +12,7 @@ dependencies:
repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
condition: solr.enabled
- name: cassandra
- version: 5.5.3
+ version: 9.1.7
repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
condition: cassandra.enabled
diff --git a/helm/values.yaml b/helm/values.yaml
index 4105362..fe2481e 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -195,8 +195,7 @@ cassandra:
dbUser:
user: cassandra
password: cassandra
- cluster:
- replicaCount: 1
+ replicaCount: 1
persistence:
storageClass: hostpath
size: 8Gi
diff --git a/tools/deletebyquery/deletebyquery.py b/tools/deletebyquery/deletebyquery.py
index 4fb7bd6..8b98111 100644
--- a/tools/deletebyquery/deletebyquery.py
+++ b/tools/deletebyquery/deletebyquery.py
@@ -262,7 +262,7 @@ def parse_args():
help='The version of the Cassandra protocol the driver should use.',
required=False,
choices=['1', '2', '3', '4', '5'],
- default='3')
+ default='4')
parser.add_argument('--solr-rows',
help='Number of rows to fetch with each Solr query to build the list of tiles to delete',
diff --git a/tools/doms-data-tools/update_doms_data_pk.py b/tools/doms-data-tools/update_doms_data_pk.py
index ed8dbe5..749995d 100644
--- a/tools/doms-data-tools/update_doms_data_pk.py
+++ b/tools/doms-data-tools/update_doms_data_pk.py
@@ -114,7 +114,7 @@ def main():
request_timeout=60.0,
)
},
- protocol_version=3,
+ protocol_version=4,
auth_provider=auth_provider) as cluster:
session = cluster.connect('doms')
diff --git a/tools/domspurge/README.md b/tools/domspurge/README.md
index 92f7cfb..e88b62f 100644
--- a/tools/domspurge/README.md
+++ b/tools/domspurge/README.md
@@ -33,5 +33,5 @@ You can build an image for this script to run it in a Kubernetes CronJob.
```shell
cd /incubator-sdap-nexus
-docker build . -f Dockerfile -t sdap-local/DomsPurge:<tag>
+docker build . -f tools/domspurge/Dockerfile -t sdap-local/DomsPurge:<tag>
```
diff --git a/tools/domspurge/purge.py b/tools/domspurge/purge.py
index 4fb2fc3..d4bb15a 100644
--- a/tools/domspurge/purge.py
+++ b/tools/domspurge/purge.py
@@ -270,7 +270,7 @@ def parse_args():
required=False,
dest='pv',
choices=['1', '2', '3', '4', '5'],
- default='3')
+ default='4')
time_before = purge_options.add_mutually_exclusive_group(required=True)