You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/06/10 20:39:04 UTC
[incubator-sdap-ingester] 03/08: make the config-operator robust to
syntactically wrong files, by default synchornization is deactivated
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit a53d08e1d24b56bf77ed3a24abb1956c721cb0b1
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Wed Jun 3 19:21:11 2020 -0700
make the config-operator robust to syntactically wrong files, by default synchornization is deactivated
---
.../config/{ConfigMap.py => K8ConfigMap.py} | 8 ++++----
sdap_ingest_manager/config/LocalDirConfig.py | 16 ++++++++++++++--
sdap_ingest_manager/config/__init__.py | 6 +++---
sdap_ingest_manager/config_operator.py | 10 ++++++----
setup.py | 10 ++++++----
tests/resources/data/collections.yml | 2 +-
tests/resources/data/dataset_config_file_ok.yml | 2 +-
7 files changed, 35 insertions(+), 19 deletions(-)
diff --git a/sdap_ingest_manager/config/ConfigMap.py b/sdap_ingest_manager/config/K8ConfigMap.py
similarity index 94%
rename from sdap_ingest_manager/config/ConfigMap.py
rename to sdap_ingest_manager/config/K8ConfigMap.py
index 4143980..156bb19 100644
--- a/sdap_ingest_manager/config/ConfigMap.py
+++ b/sdap_ingest_manager/config/K8ConfigMap.py
@@ -8,7 +8,7 @@ logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
-class ConfigMap:
+class K8ConfigMap:
def __init__(self, configmap_name, namespace, git_remote_config):
self._git_remote_config = git_remote_config
self._namespace = namespace
@@ -58,10 +58,10 @@ class ConfigMap:
finally:
return config_keys
- def _replace(self):
+ def _patch(self):
try:
logger.info(f'replace configMap entry {self._configmap_name}')
- api_response = self._api_core_v1_instance.replace_namespaced_config_map(
+ api_response = self._api_core_v1_instance.patch_namespaced_config_map(
name=self._configmap_name,
namespace=self._namespace,
body=self._create_configmap_object()
@@ -87,5 +87,5 @@ class ConfigMap:
self._create()
except ApiException as e:
logger.error("Exception when calling Kubernetes CoreV1Api ,create failed, try to replace %s\n" % e)
- self._replace()
+ self._patch()
diff --git a/sdap_ingest_manager/config/LocalDirConfig.py b/sdap_ingest_manager/config/LocalDirConfig.py
index d58f387..515ee22 100644
--- a/sdap_ingest_manager/config/LocalDirConfig.py
+++ b/sdap_ingest_manager/config/LocalDirConfig.py
@@ -1,6 +1,7 @@
import os
import time
import logging
+import yaml
from sdap_ingest_manager.config.exceptions import UnreadableFileException
@@ -25,13 +26,24 @@ class LocalDirConfig:
return files
+ def _test_read_yaml(self, file_name):
+ """ check yaml syntax raiseyaml.parser.ParserError is it doesn't"""
+ with open(os.path.join(self._local_dir, file_name), 'r') as f:
+ docs = yaml.load_all(f, Loader=yaml.FullLoader)
+ for doc in docs:
+ pass
+
def get_file_content(self, file_name):
logger.info(f'read configuration file {file_name}')
try:
- with open(os.path.join(self._local_dir, file_name)) as f:
- return f.read()
+ self._test_read_yaml(file_name)
+ with open(os.path.join(self._local_dir, file_name), 'r') as f:
+ return f.read()
except UnicodeDecodeError as e:
raise UnreadableFileException(e)
+ except yaml.parser.ParserError as e:
+ raise UnreadableFileException(e)
+
def _get_latest_update(self):
return time.ctime(max(os.path.getmtime(root) for root,_,_ in os.walk(self._local_dir)))
diff --git a/sdap_ingest_manager/config/__init__.py b/sdap_ingest_manager/config/__init__.py
index 852920f..918967c 100644
--- a/sdap_ingest_manager/config/__init__.py
+++ b/sdap_ingest_manager/config/__init__.py
@@ -1,4 +1,4 @@
from sdap_ingest_manager.config.LocalConfiguration import LocalConfiguration
-from sdap_ingest_manager.config.ConfigMap import ConfigMap
-from sdap_ingest_manager.confg.LocalDirConfig import LocalDirConfig
-from sdpa_ingest_manager.config.RemoteGitConfig import RemoteGitConfig
\ No newline at end of file
+from sdap_ingest_manager.config.K8ConfigMap import K8ConfigMap
+from sdap_ingest_manager.config.LocalDirConfig import LocalDirConfig
+from sdap_ingest_manager.config.RemoteGitConfig import RemoteGitConfig
\ No newline at end of file
diff --git a/sdap_ingest_manager/config_operator.py b/sdap_ingest_manager/config_operator.py
index fa2771a..97699b9 100644
--- a/sdap_ingest_manager/config_operator.py
+++ b/sdap_ingest_manager/config_operator.py
@@ -1,6 +1,5 @@
import argparse
-from sdap_ingest_manager.config import RemoteGitConfig, LocalDirConfig, ConfigMap
-
+from sdap_ingest_manager.config import RemoteGitConfig, LocalDirConfig, K8ConfigMap
def main():
@@ -16,6 +15,8 @@ def main():
parser.add_argument("-n", "--namespace", help="kubernetes namespace where the configuration will be deployed", required=True)
parser.add_argument("-cm", "--config-map", help="configmap name in kubernetes", required=True)
+ parser.add_argument("-u", "--updated-continuously", nargs='?', const=True, default=False,
+ help="k8 configMap is updated as soon as a syntactically correct configuration file is updated")
options = parser.parse_args()
@@ -24,10 +25,11 @@ def main():
else:
config = RemoteGitConfig(options.git_url, branch=options.git_branch, token=options.git_token)
- config_map = ConfigMap(options.config_map, options.namespace, config)
+ config_map = K8ConfigMap(options.config_map, options.namespace, config)
config_map.publish()
- config.when_updated(config_map.publish)
+ if options.updated_continuously:
+ config.when_updated(config_map.publish)
if __name__ == "__main__":
diff --git a/setup.py b/setup.py
index 75fdd72..7bd9810 100644
--- a/setup.py
+++ b/setup.py
@@ -42,9 +42,6 @@ setuptools.setup(
long_description_content_type="text/markdown",
url="https://github.com/tloubrieu-jpl/incubator-sdap-nexus-ingestion-manager",
packages=setuptools.find_packages(),
- scripts=['bin/run_collections',
- 'bin/run_single_collection',
- 'bin/run_granules'],
classifiers=[
"Programming Language :: Python :: 3",
"Operating System :: OS Independent",
@@ -53,7 +50,12 @@ setuptools.setup(
python_requires='>=3.6',
include_package_data=True,
data_files=[('.sdap_ingest_manager/resources/', ['sdap_ingest_manager/resources/dataset_config_template.yml'])],
- install_requires=pip_requirements
+ install_requires=pip_requirements,
+ entry_points={
+ 'config-operator': ['summary=sdap_ingest_manager.config_operator:main'],
+ 'collection-ingester': ['summary=sdap_ingest_manager.service:main'],
+ },
+
)
post_install_message()
diff --git a/tests/resources/data/collections.yml b/tests/resources/data/collections.yml
index 18226ba..42d2fbc 100644
--- a/tests/resources/data/collections.yml
+++ b/tests/resources/data/collections.yml
@@ -1,7 +1,7 @@
avhrr-oi-analysed-sst:
path: resources/history_manager/data/avhrr_oi/*.nc
variable: analysed_sst
- priority: 8
+ priority: 9
avhrr-oi-analysed-sst2:
path: resources/history_manager/data/avhrr_oi/*.nc
diff --git a/tests/resources/data/dataset_config_file_ok.yml b/tests/resources/data/dataset_config_file_ok.yml
index a000293..6ff0c47 100644
--- a/tests/resources/data/dataset_config_file_ok.yml
+++ b/tests/resources/data/dataset_config_file_ok.yml
@@ -7,7 +7,7 @@ ningester:
dimensions:
- lat
- lon
-
+ zobi;
---
# Tile processors configuration
ningester: