You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2022/08/04 20:00:46 UTC
[incubator-sdap-nexus] 01/01: add a script to generate collection configuration from a remote SDAP
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch create_remote_config
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 57674b29173efd343a4fee07b28e0ad116f60967
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Thu Aug 4 13:00:26 2022 -0700
add a script to generate collection configuration from a remote SDAP
---
analysis/webservice/algorithms/DataSeriesList.py | 2 +-
analysis/webservice/redirect/RemoteSDAPCache.py | 3 +-
tools/create-remote-conf/__init__.py | 0
.../create-remote-conf/create-conf-from-remote.py | 46 ++++++++++++++++++++++
tools/create-remote-conf/requirements.txt | 1 +
5 files changed, 50 insertions(+), 2 deletions(-)
diff --git a/analysis/webservice/algorithms/DataSeriesList.py b/analysis/webservice/algorithms/DataSeriesList.py
index e247bb6..4c287a9 100644
--- a/analysis/webservice/algorithms/DataSeriesList.py
+++ b/analysis/webservice/algorithms/DataSeriesList.py
@@ -33,11 +33,11 @@ class DataSeriesListCalcHandlerImpl(NexusCalcHandler):
path = "/list"
description = "Lists datasets currently available for analysis"
params = {}
- remote_sdaps = RemoteSDAPCache()
def __init__(self, tile_service_factory, remote_collections=None, **kwargs):
super().__init__(tile_service_factory, **kwargs)
self._remote_collections = remote_collections
+ self.remote_sdaps = RemoteSDAPCache()
@cached(ttl=(60 * 60 * 1000)) # 1 hour cached
diff --git a/analysis/webservice/redirect/RemoteSDAPCache.py b/analysis/webservice/redirect/RemoteSDAPCache.py
index 2568ab1..b3d9984 100644
--- a/analysis/webservice/redirect/RemoteSDAPCache.py
+++ b/analysis/webservice/redirect/RemoteSDAPCache.py
@@ -37,7 +37,8 @@ class RemoteSDAPCache:
def get(self, url, short_name):
stripped_url = url.strip('/')
- if stripped_url not in self.sdap_lists or self.sdap_lists[stripped_url].outdated_at>datetime.now():
+ logger.debug("")
+ if stripped_url not in self.sdap_lists or self.sdap_lists[stripped_url].outdated_at > datetime.now():
self._add(stripped_url)
for collection in self.sdap_lists[stripped_url].list:
diff --git a/tools/create-remote-conf/__init__.py b/tools/create-remote-conf/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tools/create-remote-conf/create-conf-from-remote.py b/tools/create-remote-conf/create-conf-from-remote.py
new file mode 100644
index 0000000..d4bec24
--- /dev/null
+++ b/tools/create-remote-conf/create-conf-from-remote.py
@@ -0,0 +1,46 @@
+import argparse
+import requests
+import yaml
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Create a collections.yaml config from a remote SDAP',
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+ parser.add_argument('--remote-nexus-list',
+ help='The url of the remote SDAP server, list end-point',
+ required=True
+ )
+
+ parser.add_argument('--prefix',
+ help='Add prefix to the remote collection id to form the local collection id',
+ required=False,
+ default="")
+
+ return parser.parse_args()
+
+
+def main():
+ the_args = parse_args()
+ r = requests.get(the_args.remote_nexus_list, verify=False)
+ collection_config = []
+ for collection in r.json():
+ if 'remoteUrl' not in collection:
+ collection_config.append(
+ {
+ 'id': the_args.prefix + collection['shortName'],
+ 'path': the_args.remote_nexus_list.removesuffix('/list'),
+ 'remote-id': collection['shortName']
+ }
+ )
+
+ with open('./collections.yml', 'w') as outfile:
+ yaml.dump(collection_config, outfile, default_flow_style=False)
+
+
+
+
+
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
diff --git a/tools/create-remote-conf/requirements.txt b/tools/create-remote-conf/requirements.txt
new file mode 100644
index 0000000..f229360
--- /dev/null
+++ b/tools/create-remote-conf/requirements.txt
@@ -0,0 +1 @@
+requests