You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2022/08/04 20:00:45 UTC
[incubator-sdap-nexus] branch create_remote_config created (now 57674b2)
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a change to branch create_remote_config
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
at 57674b2 add a script to generate collection configuration from a remote SDAP
This branch includes the following new commits:
new 57674b2 add a script to generate collection configuration from a remote SDAP
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-sdap-nexus] 01/01: add a script to generate collection configuration from a remote SDAP
Posted by tl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch create_remote_config
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 57674b29173efd343a4fee07b28e0ad116f60967
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Thu Aug 4 13:00:26 2022 -0700
add a script to generate collection configuration from a remote SDAP
---
analysis/webservice/algorithms/DataSeriesList.py | 2 +-
analysis/webservice/redirect/RemoteSDAPCache.py | 3 +-
tools/create-remote-conf/__init__.py | 0
.../create-remote-conf/create-conf-from-remote.py | 46 ++++++++++++++++++++++
tools/create-remote-conf/requirements.txt | 1 +
5 files changed, 50 insertions(+), 2 deletions(-)
diff --git a/analysis/webservice/algorithms/DataSeriesList.py b/analysis/webservice/algorithms/DataSeriesList.py
index e247bb6..4c287a9 100644
--- a/analysis/webservice/algorithms/DataSeriesList.py
+++ b/analysis/webservice/algorithms/DataSeriesList.py
@@ -33,11 +33,11 @@ class DataSeriesListCalcHandlerImpl(NexusCalcHandler):
path = "/list"
description = "Lists datasets currently available for analysis"
params = {}
- remote_sdaps = RemoteSDAPCache()
def __init__(self, tile_service_factory, remote_collections=None, **kwargs):
super().__init__(tile_service_factory, **kwargs)
self._remote_collections = remote_collections
+ self.remote_sdaps = RemoteSDAPCache()
@cached(ttl=(60 * 60 * 1000)) # 1 hour cached
diff --git a/analysis/webservice/redirect/RemoteSDAPCache.py b/analysis/webservice/redirect/RemoteSDAPCache.py
index 2568ab1..b3d9984 100644
--- a/analysis/webservice/redirect/RemoteSDAPCache.py
+++ b/analysis/webservice/redirect/RemoteSDAPCache.py
@@ -37,7 +37,8 @@ class RemoteSDAPCache:
def get(self, url, short_name):
stripped_url = url.strip('/')
- if stripped_url not in self.sdap_lists or self.sdap_lists[stripped_url].outdated_at>datetime.now():
+ logger.debug("")
+ if stripped_url not in self.sdap_lists or self.sdap_lists[stripped_url].outdated_at > datetime.now():
self._add(stripped_url)
for collection in self.sdap_lists[stripped_url].list:
diff --git a/tools/create-remote-conf/__init__.py b/tools/create-remote-conf/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/create-remote-conf/create-conf-from-remote.py b/tools/create-remote-conf/create-conf-from-remote.py
new file mode 100644
index 0000000..d4bec24
--- /dev/null
+++ b/tools/create-remote-conf/create-conf-from-remote.py
@@ -0,0 +1,46 @@
+import argparse
+import requests
+import yaml
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Create a collections.yaml config from a remote SDAP',
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+ parser.add_argument('--remote-nexus-list',
+ help='The url of the remote SDAP server, list end-point',
+ required=True
+ )
+
+ parser.add_argument('--prefix',
+ help='Add prefix to the remote collection id to form the local collection id',
+ required=False,
+ default="")
+
+ return parser.parse_args()
+
+
+def main():
+ the_args = parse_args()
+ r = requests.get(the_args.remote_nexus_list, verify=False)
+ collection_config = []
+ for collection in r.json():
+ if 'remoteUrl' not in collection:
+ collection_config.append(
+ {
+ 'id': the_args.prefix + collection['shortName'],
+ 'path': the_args.remote_nexus_list.removesuffix('/list'),
+ 'remote-id': collection['shortName']
+ }
+ )
+
+ with open('./collections.yml', 'w') as outfile:
+ yaml.dump(collection_config, outfile, default_flow_style=False)
+
+
+
+
+
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
diff --git a/tools/create-remote-conf/requirements.txt b/tools/create-remote-conf/requirements.txt
new file mode 100644
index 0000000..f229360
--- /dev/null
+++ b/tools/create-remote-conf/requirements.txt
@@ -0,0 +1 @@
+requests