You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/06/10 20:39:01 UTC
[incubator-sdap-ingester] branch config_map created (now 7a655f0)
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a change to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git.
at 7a655f0 apply change requests by pull request #3 review
This branch includes the following new commits:
new 5520d27 initiate a configMap object with unit test (with bug)
new 5fa6fb4 add config_operator working on local dir or remote git repo
new a53d08e make the config-operator robust to syntactically wrong files, by default synchornization is deactivated
new b828121 start to implement config-operator containers
new 49042bb move files in specific subdirectory, start k8s deployment, does not work
new 3268385 integrates with kubernetes
new 1fbfce5 run config operator as a k8s operator
new 7a655f0 apply change requests by pull request #3 review
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-sdap-ingester] 08/08: apply change requests by pull
request #3 review
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 7a655f049c0191aed40352de9162f9d27f1793fb
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Mon Jun 8 19:48:44 2020 -0700
apply change requests by pull request #3 review
---
config_operator/README.md | 2 ++
.../config_source/LocalDirConfig.py | 18 +++++++++----
.../config_source/RemoteGitConfig.py | 30 +++++++++++++---------
.../config_operator/k8s/K8sConfigMap.py | 13 +++++++---
config_operator/config_operator/main.py | 14 +++++++---
config_operator/containers/docker/Dockerfile | 4 +--
.../containers/k8s/config-operator-crd.yml | 2 ++
.../tests/{config => config_source}/__init__.py | 0
config_operator/tests/{config => k8s}/__init__.py | 0
.../test_ConfigMap.py => k8s/test_K8sConfigMap.py} | 2 +-
10 files changed, 58 insertions(+), 27 deletions(-)
diff --git a/config_operator/README.md b/config_operator/README.md
index 2e1ea0d..2ff8072 100644
--- a/config_operator/README.md
+++ b/config_operator/README.md
@@ -51,4 +51,6 @@ Check that the configMap has been generated:
kubectl get configmaps -n sdap
+
+
\ No newline at end of file
diff --git a/config_operator/config_operator/config_source/LocalDirConfig.py b/config_operator/config_operator/config_source/LocalDirConfig.py
index f37e41b..cf95f42 100644
--- a/config_operator/config_operator/config_source/LocalDirConfig.py
+++ b/config_operator/config_operator/config_source/LocalDirConfig.py
@@ -2,6 +2,7 @@ import os
import time
import logging
import yaml
+from typing import Callable
from config_operator.config_source.exceptions import UnreadableFileException
@@ -14,10 +15,12 @@ LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1
class LocalDirConfig:
- def __init__(self, local_dir):
+ def __init__(self, local_dir: str,
+ update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS):
logger.info(f'create config on local dir {local_dir}')
self._local_dir = local_dir
self._latest_update = self._get_latest_update()
+ self._update_every_seconds=update_every_seconds
def get_files(self):
files = []
@@ -29,14 +32,14 @@ class LocalDirConfig:
return files
- def _test_read_yaml(self, file_name):
+ def _test_read_yaml(self, file_name: str):
""" check yaml syntax raise yaml.parser.ParserError is it doesn't"""
with open(os.path.join(self._local_dir, file_name), 'r') as f:
docs = yaml.load_all(f, Loader=yaml.FullLoader)
for doc in docs:
pass
- def get_file_content(self, file_name):
+ def get_file_content(self, file_name: str):
logger.info(f'read configuration file {file_name}')
try:
self._test_read_yaml(file_name)
@@ -55,9 +58,12 @@ class LocalDirConfig:
else:
return None
- def when_updated(self, callback):
+ def when_updated(self, callback: Callable[[], None]):
+ """
+ call function callback when the local directory is updated.
+ """
while True:
- time.sleep(LISTEN_FOR_UPDATE_INTERVAL_SECONDS)
+ time.sleep(self._update_every_seconds)
latest_update = self._get_latest_update()
if latest_update is None or (latest_update > self._latest_update):
logger.info("local config dir has been updated")
@@ -66,3 +72,5 @@ class LocalDirConfig:
else:
logger.debug("local config dir has not been updated")
+ return None
+
diff --git a/config_operator/config_operator/config_source/RemoteGitConfig.py b/config_operator/config_operator/config_source/RemoteGitConfig.py
index 15c0f01..17d8223 100644
--- a/config_operator/config_operator/config_source/RemoteGitConfig.py
+++ b/config_operator/config_operator/config_source/RemoteGitConfig.py
@@ -3,6 +3,7 @@ import os
import sys
import time
from git import Repo
+from typing import Callable
from .LocalDirConfig import LocalDirConfig
logging.basicConfig(level=logging.DEBUG)
@@ -11,11 +12,13 @@ logger = logging.getLogger(__name__)
LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 5
DEFAULT_LOCAL_REPO_DIR = os.path.join(sys.prefix, 'sdap', 'conf')
+
class RemoteGitConfig(LocalDirConfig):
- def __init__(self, git_url,
- branch='master',
- token=None,
- local_dir=DEFAULT_LOCAL_REPO_DIR
+ def __init__(self, git_url: str,
+ git_branch: str = 'master',
+ git_token: str = None,
+ update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS,
+ local_dir: str = DEFAULT_LOCAL_REPO_DIR
):
"""
@@ -24,11 +27,12 @@ class RemoteGitConfig(LocalDirConfig):
:param git_token:
"""
self._git_url = git_url if git_url.endswith(".git") else git_url + '.git'
- self._git_branch = branch
- self._git_token = token
+ self._git_branch = git_branch
+ self._git_token = git_token
if local_dir is None:
local_dir = DEFAULT_LOCAL_REPO_DIR
- super().__init__(local_dir)
+ self._update_every_seconds = update_every_seconds
+ super().__init__(local_dir, update_every_seconds=self._update_every_seconds)
self._repo = None
self._init_local_config_repo()
self._latest_commit_key = self._pull_remote()
@@ -45,12 +49,12 @@ class RemoteGitConfig(LocalDirConfig):
self._repo.git.fetch()
self._repo.git.checkout(self._git_branch)
-
-
- def when_updated(self, callback):
-
+ def when_updated(self, callback: Callable[[], None]):
+ """
+ call function callback when the remote git repository is updated.
+ """
while True:
- time.sleep(LISTEN_FOR_UPDATE_INTERVAL_SECONDS)
+ time.sleep(self._update_every_seconds)
remote_commit_key = self._pull_remote()
if remote_commit_key != self._latest_commit_key:
logger.info("remote git repository has been updated")
@@ -59,3 +63,5 @@ class RemoteGitConfig(LocalDirConfig):
else:
logger.debug("remote git repository has not been updated")
+ return None
+
diff --git a/config_operator/config_operator/k8s/K8sConfigMap.py b/config_operator/config_operator/k8s/K8sConfigMap.py
index 32ee844..e2a7a10 100644
--- a/config_operator/config_operator/k8s/K8sConfigMap.py
+++ b/config_operator/config_operator/k8s/K8sConfigMap.py
@@ -1,7 +1,9 @@
import os
import logging
from kubernetes import client, config
+from config_operator.config_source import LocalDirConfig, RemoteGitConfig
from kubernetes.client.rest import ApiException
+from typing import Union
from config_operator.config_source.exceptions import UnreadableFileException
@@ -10,11 +12,14 @@ logger = logging.getLogger(__name__)
class K8sConfigMap:
- def __init__(self, configmap_name, namespace, git_remote_config):
- self._git_remote_config = git_remote_config
+ def __init__(self, configmap_name: str,
+ namespace: str,
+ external_config: Union[LocalDirConfig, RemoteGitConfig]):
+ self._git_remote_config = external_config
self._namespace = namespace
self._configmap_name = configmap_name
+ # test is this runs inside kubernetes cluster
if os.getenv('KUBERNETES_SERVICE_HOST'):
config.load_incluster_config()
else:
@@ -93,6 +98,8 @@ class K8sConfigMap:
try:
self._create()
except ApiException as e:
- logger.error("Exception when calling Kubernetes CoreV1Api ,create failed, try to replace %s\n" % e)
+ logger.error("Exception when calling Kubernetes CoreV1Api ,create failed, try to patch %s\n" % e)
self._patch()
+ return None
+
diff --git a/config_operator/config_operator/main.py b/config_operator/config_operator/main.py
index 3d6ad2e..db4dbcb 100644
--- a/config_operator/config_operator/main.py
+++ b/config_operator/config_operator/main.py
@@ -1,9 +1,12 @@
+import logging
import kopf
from config_operator.config_source import RemoteGitConfig
from config_operator.k8s import K8sConfigMap
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
-@kopf.on.create('sdap.apache.org', 'v1', 'git-repo-configs')
+@kopf.on.create('sdap.apache.org', 'v1', 'gitbasedconfigs')
def create_fn(body, spec, **kwargs):
# Get info from Git Repo Config object
name = body['metadata']['name']
@@ -15,12 +18,15 @@ def create_fn(body, spec, **kwargs):
raise kopf.HandlerFatalError(f"config-map must be set.")
git_url = spec['git-url']
+ logger.info(f'git-url = {git_url}')
config_map = spec['config-map']
+ logger.info(f'config-map = {config_map}')
_kargs = {}
- for k in {'git-branch', 'git-token'}:
- if k in spec.keys():
- _kargs[k.split('-')[0]] = spec[k]
+ for k in {'git-branch', 'git-token', 'update-every-seconds'}:
+ if k in spec:
+ logger.info(f'{k} = {spec[k]}')
+ _kargs[k.replace('-', '_')] = spec[k]
config = RemoteGitConfig(git_url, **_kargs)
diff --git a/config_operator/containers/docker/Dockerfile b/config_operator/containers/docker/Dockerfile
index 4765d30..a161dae 100644
--- a/config_operator/containers/docker/Dockerfile
+++ b/config_operator/containers/docker/Dockerfile
@@ -6,6 +6,6 @@ COPY /requirements.txt /config_operator/requirements.txt
COPY /README.md /config_operator/README.md
RUN cd /config_operator && pip install .
-COPY /config_operator/k8_config_operator.py /k8_config_operator.py
+COPY /config_operator/main.py /main.py
-CMD ["kopf", "run", "/k8_config_operator.py", "--verbose"]
+CMD ["kopf", "run", "/main.py", "--verbose"]
diff --git a/config_operator/containers/k8s/config-operator-crd.yml b/config_operator/containers/k8s/config-operator-crd.yml
index ea9bbba..9f6d2ad 100644
--- a/config_operator/containers/k8s/config-operator-crd.yml
+++ b/config_operator/containers/k8s/config-operator-crd.yml
@@ -25,6 +25,8 @@ spec:
type: string
git-token:
type: string
+ update-every-seconds:
+ type: int
local-dir:
type: string
namespace:
diff --git a/config_operator/tests/config/__init__.py b/config_operator/tests/config_source/__init__.py
similarity index 100%
copy from config_operator/tests/config/__init__.py
copy to config_operator/tests/config_source/__init__.py
diff --git a/config_operator/tests/config/__init__.py b/config_operator/tests/k8s/__init__.py
similarity index 100%
rename from config_operator/tests/config/__init__.py
rename to config_operator/tests/k8s/__init__.py
diff --git a/config_operator/tests/config/test_ConfigMap.py b/config_operator/tests/k8s/test_K8sConfigMap.py
similarity index 95%
rename from config_operator/tests/config/test_ConfigMap.py
rename to config_operator/tests/k8s/test_K8sConfigMap.py
index 4d339e3..1660e69 100644
--- a/config_operator/tests/config/test_ConfigMap.py
+++ b/config_operator/tests/k8s/test_K8sConfigMap.py
@@ -5,7 +5,7 @@ from config_operator.k8s import K8sConfigMap
from config_operator.config_source import RemoteGitConfig, LocalDirConfig
-class ConfigMapTest(unittest.TestCase):
+class K8sConfigMapTest(unittest.TestCase):
def test_createconfigmapfromgit(self):
remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config")
[incubator-sdap-ingester] 05/08: move files in specific
subdirectory, start k8s deployment, does not work
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 49042bb822c2c2e9ab3eb26dd2d07e4b2992f383
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Thu Jun 4 20:19:42 2020 -0700
move files in specific subdirectory, start k8s deployment, does not work
---
README.md | 11 -------
config_operator/README.md | 38 ++++++++++++++++++++++
config_operator/config_operator/__init__.py | 1 +
.../config_operator}/config_operator.py | 5 +--
.../config_source}/LocalDirConfig.py | 5 ++-
.../config_source}/RemoteGitConfig.py | 27 +++++++--------
.../config_operator/config_source/__init__.py | 2 ++
.../config_operator/config_source}/exceptions.py | 2 +-
.../config_operator/k8s/K8sConfigMap.py | 4 +--
config_operator/config_operator/k8s/__init__.py | 1 +
config_operator/containers/docker/Dockerfile | 10 ++++++
config_operator/containers/k8s/deployment.yml | 21 ++++++++++++
config_operator/requirements.txt | 3 ++
setup.py => config_operator/setup.py | 16 ++++-----
.../config => config_operator/tests}/__init__.py | 0
.../tests}/config/__init__.py | 0
config_operator/tests/config/test_ConfigMap.py | 27 +++++++++++++++
config_operator/tests/resources/collections.yml | 9 +++++
containers/docker/config-operator/Dockerfile | 10 ------
sdap_ingest_manager/config/__init__.py | 3 --
setup.py | 2 +-
tests/config/test_ConfigMap.py | 22 -------------
22 files changed, 141 insertions(+), 78 deletions(-)
diff --git a/README.md b/README.md
index d653212..adb6ddd 100644
--- a/README.md
+++ b/README.md
@@ -128,17 +128,6 @@ Deploy a local rabbitmq service, for example with docker.
### Launch the service
-#### The config operator:
-
-This component helps to import a configuration directory on local file system or on a git repository as a configMap in kubernetes.
-This makes the configuration easily accessible to all the nodes of the cluster whereas the configuration stays in a single place.
-The configurations can be updated while the service is running (-u). The configuration updates will be published to kubernetes pods by patching the existing configurations.
-
- config-operator -h
- config-operator -l tests/resources/data -n sdap -cm collection-ingester-config
- config-operator --git-url=https://github.com/tloubrieu-jpl/sdap-ingester-config --namespace=sdap --config-map=collection-ingester-config
-
-#### The collection ingestion service
The service reads the collection configuration and submit granule ingestion messages to the message broker (rabbitmq).
For each collection, 2 ingestion priority levels are proposed: the nominal priority, the priority for forward processing (newer files), usually higher.
diff --git a/config_operator/README.md b/config_operator/README.md
new file mode 100644
index 0000000..61889c4
--- /dev/null
+++ b/config_operator/README.md
@@ -0,0 +1,38 @@
+# Config Operator
+
+## Purpose
+
+Component which synchonizes local configuration in a directory, on a file system, or configuration files managed in a git repository with kubernetes configMap.
+This helps to make a configuration managed by the operators in a single place (git, host file system) available in the kubernetes cluster.
+
+For SDAP, it is used to make the configuration of the collections to be ingested available to the ingester service pods.
+
+# Launch the service
+
+The configurations can be updated while the service is running (-u). The configuration updates will be published to kubernetes pods by patching the existing configurations.
+
+ config-operator -h
+ config-operator -l tests/resources/data -n sdap -cm collection-ingester-config
+ config-operator --git-url=https://github.com/tloubrieu-jpl/sdap-ingester-config --namespace=sdap --config-map=collection-ingester-config
+
+# Developers
+
+ git clone ...
+ cd config_operator
+ pip install -e .
+ pytest -d
+
+# Containerizaion
+
+## Docker
+
+ docker build . -f containers/docker/Dockerfile --no-cache --tag tloubrieu/config-operator:latest
+
+To publish the docker image on dockerhub do (step necessary for kubernetes deployment):
+
+ docker login
+ docker push tloubrieu/sdap-ingest-manager:latest
+
+## Kubernetes
+
+ kubectl apply -f containers/k8s/deployment.yml -n sdap
\ No newline at end of file
diff --git a/config_operator/config_operator/__init__.py b/config_operator/config_operator/__init__.py
new file mode 100644
index 0000000..7d24cef
--- /dev/null
+++ b/config_operator/config_operator/__init__.py
@@ -0,0 +1 @@
+__version__ = '0.1.0.dev0'
diff --git a/sdap_ingest_manager/config_operator.py b/config_operator/config_operator/config_operator.py
similarity index 89%
rename from sdap_ingest_manager/config_operator.py
rename to config_operator/config_operator/config_operator.py
index 97699b9..b032384 100644
--- a/sdap_ingest_manager/config_operator.py
+++ b/config_operator/config_operator/config_operator.py
@@ -1,5 +1,6 @@
import argparse
-from sdap_ingest_manager.config import RemoteGitConfig, LocalDirConfig, K8ConfigMap
+from config_operator.config_source import RemoteGitConfig, LocalDirConfig
+from config_operator.k8s import K8sConfigMap
def main():
@@ -25,7 +26,7 @@ def main():
else:
config = RemoteGitConfig(options.git_url, branch=options.git_branch, token=options.git_token)
- config_map = K8ConfigMap(options.config_map, options.namespace, config)
+ config_map = K8sConfigMap(options.config_map, options.namespace, config)
config_map.publish()
if options.updated_continuously:
diff --git a/sdap_ingest_manager/config/LocalDirConfig.py b/config_operator/config_operator/config_source/LocalDirConfig.py
similarity index 93%
rename from sdap_ingest_manager/config/LocalDirConfig.py
rename to config_operator/config_operator/config_source/LocalDirConfig.py
index 6d8a7cf..89c0e5a 100644
--- a/sdap_ingest_manager/config/LocalDirConfig.py
+++ b/config_operator/config_operator/config_source/LocalDirConfig.py
@@ -3,7 +3,8 @@ import time
import logging
import yaml
-from sdap_ingest_manager.config.exceptions import UnreadableFileException
+
+from config_operator.config_source.exceptions import UnreadableFileException
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
@@ -12,7 +13,9 @@ LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1
class LocalDirConfig:
+
def __init__(self, local_dir):
+ logger.info(f'create config on local dir {local_dir}')
self._local_dir = local_dir
self._latest_update = self._get_latest_update()
diff --git a/sdap_ingest_manager/config/RemoteGitConfig.py b/config_operator/config_operator/config_source/RemoteGitConfig.py
similarity index 77%
rename from sdap_ingest_manager/config/RemoteGitConfig.py
rename to config_operator/config_operator/config_source/RemoteGitConfig.py
index a344246..24e614a 100644
--- a/sdap_ingest_manager/config/RemoteGitConfig.py
+++ b/config_operator/config_operator/config_source/RemoteGitConfig.py
@@ -2,19 +2,19 @@ import logging
import os
import sys
import time
-from git import Repo, Remote
-from sdap_ingest_manager.config import LocalDirConfig
-
+from git import Repo
+from .LocalDirConfig import LocalDirConfig
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 5
+
class RemoteGitConfig(LocalDirConfig):
def __init__(self, git_url,
- git_branch='master',
- git_token=None
+ branch='master',
+ token=None
):
"""
@@ -23,18 +23,18 @@ class RemoteGitConfig(LocalDirConfig):
:param git_token:
"""
self._git_url = git_url if git_url.endswith(".git") else git_url + '.git'
- self._git_branch = git_branch
- self._git_token = git_token
+ self._git_branch = branch
+ self._git_token = token
local_dir = os.path.join(sys.prefix, 'sdap', 'conf')
super().__init__(local_dir)
self._repo = None
self._init_local_config_repo()
- self._latest_commit_key = self._repo.head.commit.hexsha
+ self._latest_commit_key = self._pull_remote()
def _pull_remote(self):
o = self._repo.remotes.origin
res = o.pull()
- return res[0].commit.hexsha # return the latest commit key
+ return res[0].commit.hexsha # return the latest commit key
def _init_local_config_repo(self):
self._repo = Repo.init(self._local_dir)
@@ -43,6 +43,8 @@ class RemoteGitConfig(LocalDirConfig):
self._repo.git.fetch()
self._repo.git.checkout(self._git_branch)
+
+
def when_updated(self, callback):
while True:
@@ -54,9 +56,4 @@ class RemoteGitConfig(LocalDirConfig):
self._latest_commit_key = remote_commit_key
else:
logger.debug("remote git repository has not been updated")
-
-
-
-
-
-
+
diff --git a/config_operator/config_operator/config_source/__init__.py b/config_operator/config_operator/config_source/__init__.py
new file mode 100644
index 0000000..b23e9be
--- /dev/null
+++ b/config_operator/config_operator/config_source/__init__.py
@@ -0,0 +1,2 @@
+from .RemoteGitConfig import RemoteGitConfig
+from .LocalDirConfig import LocalDirConfig
diff --git a/sdap_ingest_manager/config/exceptions.py b/config_operator/config_operator/config_source/exceptions.py
similarity index 83%
rename from sdap_ingest_manager/config/exceptions.py
rename to config_operator/config_operator/config_source/exceptions.py
index c06b881..96e8502 100644
--- a/sdap_ingest_manager/config/exceptions.py
+++ b/config_operator/config_operator/config_source/exceptions.py
@@ -1,4 +1,4 @@
class UnreadableFileException(Exception):
- pass
\ No newline at end of file
+ pass
diff --git a/sdap_ingest_manager/config/K8ConfigMap.py b/config_operator/config_operator/k8s/K8sConfigMap.py
similarity index 96%
rename from sdap_ingest_manager/config/K8ConfigMap.py
rename to config_operator/config_operator/k8s/K8sConfigMap.py
index e9afe55..b16b58c 100644
--- a/sdap_ingest_manager/config/K8ConfigMap.py
+++ b/config_operator/config_operator/k8s/K8sConfigMap.py
@@ -2,13 +2,13 @@ import logging
from kubernetes import client, config
from kubernetes.client.rest import ApiException
-from sdap_ingest_manager.config.exceptions import UnreadableFileException
+from config_operator.config_source.exceptions import UnreadableFileException
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
-class K8ConfigMap:
+class K8sConfigMap:
def __init__(self, configmap_name, namespace, git_remote_config):
self._git_remote_config = git_remote_config
self._namespace = namespace
diff --git a/config_operator/config_operator/k8s/__init__.py b/config_operator/config_operator/k8s/__init__.py
new file mode 100644
index 0000000..2d5a84d
--- /dev/null
+++ b/config_operator/config_operator/k8s/__init__.py
@@ -0,0 +1 @@
+from .K8sConfigMap import K8sConfigMap
diff --git a/config_operator/containers/docker/Dockerfile b/config_operator/containers/docker/Dockerfile
new file mode 100644
index 0000000..4e82c98
--- /dev/null
+++ b/config_operator/containers/docker/Dockerfile
@@ -0,0 +1,10 @@
+FROM python:3
+
+COPY /config_operator /config_operator/config_operator
+COPY /setup.py /config_operator/setup.py
+COPY /requirements.txt /config_operator/requirements.txt
+COPY /README.md /config_operator/README.md
+
+RUN cd /config_operator && pip install .
+
+CMD bash
diff --git a/config_operator/containers/k8s/deployment.yml b/config_operator/containers/k8s/deployment.yml
new file mode 100644
index 0000000..1501a75
--- /dev/null
+++ b/config_operator/containers/k8s/deployment.yml
@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: config-operator
+ labels:
+ app: sdap-config-operator
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: sdap-config-operator
+ template:
+ metadata:
+ labels:
+ app: sdap-config-operator
+ spec:
+ containers:
+ - name: sdap-config-operator
+ image: tloubrieu/config-operator:latest
+ imagePullPolicy: IfNotPresent
+ command: ['config-operator', '--git-url', 'https://github.com/tloubrieu-jpl/sdap-ingester-config' , '--namespace', 'sdap', '--config-map', 'collection-ingester-conf', '-u']
diff --git a/config_operator/requirements.txt b/config_operator/requirements.txt
new file mode 100644
index 0000000..4365d3d
--- /dev/null
+++ b/config_operator/requirements.txt
@@ -0,0 +1,3 @@
+GitPython==3.1.2
+kubernetes==11.0
+
diff --git a/setup.py b/config_operator/setup.py
similarity index 65%
copy from setup.py
copy to config_operator/setup.py
index f85d6a7..2d4c533 100644
--- a/setup.py
+++ b/config_operator/setup.py
@@ -1,15 +1,13 @@
import setuptools
-import os
-import subprocess
-import sys
import re
-PACKAGE_NAME = "sdap_ingest_manager"
+PACKAGE_NAME = "config_operator"
-with open("./sdap_ingest_manager/__init__.py") as fi:
+with open(f'./{PACKAGE_NAME}/__init__.py') as fi:
result = re.search(r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fi.read())
version = result.group(1)
+
with open("README.md", "r") as fh:
long_description = fh.read()
@@ -21,7 +19,7 @@ setuptools.setup(
version=version,
author="Apache - SDAP",
author_email="dev@sdap.apache.org",
- description="a helper to ingest data in sdap",
+ description="a service to synchronize git or local directory configuration with k8s configMap",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/tloubrieu-jpl/incubator-sdap-nexus-ingestion-manager",
@@ -33,10 +31,8 @@ setuptools.setup(
],
python_requires='>=3.6',
include_package_data=True,
- data_files=[('.sdap_ingest_manager/resources/', ['sdap_ingest_manager/resources/dataset_config_template.yml'])],
install_requires=requirements,
entry_points={
- 'console_scripts': ['config-operator=sdap_ingest_manager.config_operator:main',
- 'collection-ingester=sdap_ingest_manager.service:main']
- },
+ 'console_scripts': ['config-operator=config_operator.config_operator:main']
+ }
)
diff --git a/tests/config/__init__.py b/config_operator/tests/__init__.py
similarity index 100%
copy from tests/config/__init__.py
copy to config_operator/tests/__init__.py
diff --git a/tests/config/__init__.py b/config_operator/tests/config/__init__.py
similarity index 100%
rename from tests/config/__init__.py
rename to config_operator/tests/config/__init__.py
diff --git a/config_operator/tests/config/test_ConfigMap.py b/config_operator/tests/config/test_ConfigMap.py
new file mode 100644
index 0000000..4d339e3
--- /dev/null
+++ b/config_operator/tests/config/test_ConfigMap.py
@@ -0,0 +1,27 @@
+import unittest
+import os
+
+from config_operator.k8s import K8sConfigMap
+from config_operator.config_source import RemoteGitConfig, LocalDirConfig
+
+
+class ConfigMapTest(unittest.TestCase):
+ def test_createconfigmapfromgit(self):
+
+ remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config")
+
+ config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config)
+ config_map.publish()
+
+ def test_createconfigmapfromlocaldir(self):
+ local_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ '..',
+ 'resources')
+ remote_git_config = LocalDirConfig(local_dir)
+
+ config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config)
+ config_map.publish()
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/config_operator/tests/resources/collections.yml b/config_operator/tests/resources/collections.yml
new file mode 100644
index 0000000..42d2fbc
--- /dev/null
+++ b/config_operator/tests/resources/collections.yml
@@ -0,0 +1,9 @@
+avhrr-oi-analysed-sst:
+ path: resources/history_manager/data/avhrr_oi/*.nc
+ variable: analysed_sst
+ priority: 9
+
+avhrr-oi-analysed-sst2:
+ path: resources/history_manager/data/avhrr_oi/*.nc
+ variable: analysed_sst
+ priority: 1
diff --git a/containers/docker/config-operator/Dockerfile b/containers/docker/config-operator/Dockerfile
deleted file mode 100644
index 81918b8..0000000
--- a/containers/docker/config-operator/Dockerfile
+++ /dev/null
@@ -1,10 +0,0 @@
-FROM python:3
-
-COPY /sdap_ingest_manager /sdap_ingest_manager/sdap_ingest_manager
-COPY /setup.py /sdap_ingest_manager/setup.py
-COPY /requirements.txt /sdap_ingest_manager/requirements.txt
-COPY /README.md /sdap_ingest_manager/README.md
-
-RUN cd /sdap_ingest_manager && pip install .
-
-CMD bash
diff --git a/sdap_ingest_manager/config/__init__.py b/sdap_ingest_manager/config/__init__.py
index 918967c..27a89b8 100644
--- a/sdap_ingest_manager/config/__init__.py
+++ b/sdap_ingest_manager/config/__init__.py
@@ -1,4 +1 @@
from sdap_ingest_manager.config.LocalConfiguration import LocalConfiguration
-from sdap_ingest_manager.config.K8ConfigMap import K8ConfigMap
-from sdap_ingest_manager.config.LocalDirConfig import LocalDirConfig
-from sdap_ingest_manager.config.RemoteGitConfig import RemoteGitConfig
\ No newline at end of file
diff --git a/setup.py b/setup.py
index f85d6a7..eeed194 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@ setuptools.setup(
data_files=[('.sdap_ingest_manager/resources/', ['sdap_ingest_manager/resources/dataset_config_template.yml'])],
install_requires=requirements,
entry_points={
- 'console_scripts': ['config-operator=sdap_ingest_manager.config_operator:main',
+ 'console_scripts': ['config-operator=config_operator.config_operator:main',
'collection-ingester=sdap_ingest_manager.service:main']
},
)
diff --git a/tests/config/test_ConfigMap.py b/tests/config/test_ConfigMap.py
deleted file mode 100644
index 2518b9a..0000000
--- a/tests/config/test_ConfigMap.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import unittest
-import os
-
-from flask import Flask
-from flask_restplus import Api
-
-from sdap_ingest_manager.config.ConfigMap import ConfigMap
-from sdap_ingest_manager.config.RemoteGitConfig import RemoteGitConfig
-
-
-class ConfigMapTest(unittest.TestCase):
- def test_createconfigmap(self):
-
- remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config")
-
- config_map = ConfigMap('collection-ingester', 'sdap', remote_git_config)
- config_map.publish()
-
-
-
-if __name__ == '__main__':
- unittest.main()
[incubator-sdap-ingester] 03/08: make the config-operator robust to
syntactically wrong files, by default synchornization is deactivated
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit a53d08e1d24b56bf77ed3a24abb1956c721cb0b1
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Wed Jun 3 19:21:11 2020 -0700
make the config-operator robust to syntactically wrong files, by default synchornization is deactivated
---
.../config/{ConfigMap.py => K8ConfigMap.py} | 8 ++++----
sdap_ingest_manager/config/LocalDirConfig.py | 16 ++++++++++++++--
sdap_ingest_manager/config/__init__.py | 6 +++---
sdap_ingest_manager/config_operator.py | 10 ++++++----
setup.py | 10 ++++++----
tests/resources/data/collections.yml | 2 +-
tests/resources/data/dataset_config_file_ok.yml | 2 +-
7 files changed, 35 insertions(+), 19 deletions(-)
diff --git a/sdap_ingest_manager/config/ConfigMap.py b/sdap_ingest_manager/config/K8ConfigMap.py
similarity index 94%
rename from sdap_ingest_manager/config/ConfigMap.py
rename to sdap_ingest_manager/config/K8ConfigMap.py
index 4143980..156bb19 100644
--- a/sdap_ingest_manager/config/ConfigMap.py
+++ b/sdap_ingest_manager/config/K8ConfigMap.py
@@ -8,7 +8,7 @@ logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
-class ConfigMap:
+class K8ConfigMap:
def __init__(self, configmap_name, namespace, git_remote_config):
self._git_remote_config = git_remote_config
self._namespace = namespace
@@ -58,10 +58,10 @@ class ConfigMap:
finally:
return config_keys
- def _replace(self):
+ def _patch(self):
try:
logger.info(f'replace configMap entry {self._configmap_name}')
- api_response = self._api_core_v1_instance.replace_namespaced_config_map(
+ api_response = self._api_core_v1_instance.patch_namespaced_config_map(
name=self._configmap_name,
namespace=self._namespace,
body=self._create_configmap_object()
@@ -87,5 +87,5 @@ class ConfigMap:
self._create()
except ApiException as e:
logger.error("Exception when calling Kubernetes CoreV1Api ,create failed, try to replace %s\n" % e)
- self._replace()
+ self._patch()
diff --git a/sdap_ingest_manager/config/LocalDirConfig.py b/sdap_ingest_manager/config/LocalDirConfig.py
index d58f387..515ee22 100644
--- a/sdap_ingest_manager/config/LocalDirConfig.py
+++ b/sdap_ingest_manager/config/LocalDirConfig.py
@@ -1,6 +1,7 @@
import os
import time
import logging
+import yaml
from sdap_ingest_manager.config.exceptions import UnreadableFileException
@@ -25,13 +26,24 @@ class LocalDirConfig:
return files
+ def _test_read_yaml(self, file_name):
+ """ check yaml syntax raiseyaml.parser.ParserError is it doesn't"""
+ with open(os.path.join(self._local_dir, file_name), 'r') as f:
+ docs = yaml.load_all(f, Loader=yaml.FullLoader)
+ for doc in docs:
+ pass
+
def get_file_content(self, file_name):
logger.info(f'read configuration file {file_name}')
try:
- with open(os.path.join(self._local_dir, file_name)) as f:
- return f.read()
+ self._test_read_yaml(file_name)
+ with open(os.path.join(self._local_dir, file_name), 'r') as f:
+ return f.read()
except UnicodeDecodeError as e:
raise UnreadableFileException(e)
+ except yaml.parser.ParserError as e:
+ raise UnreadableFileException(e)
+
def _get_latest_update(self):
return time.ctime(max(os.path.getmtime(root) for root,_,_ in os.walk(self._local_dir)))
diff --git a/sdap_ingest_manager/config/__init__.py b/sdap_ingest_manager/config/__init__.py
index 852920f..918967c 100644
--- a/sdap_ingest_manager/config/__init__.py
+++ b/sdap_ingest_manager/config/__init__.py
@@ -1,4 +1,4 @@
from sdap_ingest_manager.config.LocalConfiguration import LocalConfiguration
-from sdap_ingest_manager.config.ConfigMap import ConfigMap
-from sdap_ingest_manager.confg.LocalDirConfig import LocalDirConfig
-from sdpa_ingest_manager.config.RemoteGitConfig import RemoteGitConfig
\ No newline at end of file
+from sdap_ingest_manager.config.K8ConfigMap import K8ConfigMap
+from sdap_ingest_manager.config.LocalDirConfig import LocalDirConfig
+from sdap_ingest_manager.config.RemoteGitConfig import RemoteGitConfig
\ No newline at end of file
diff --git a/sdap_ingest_manager/config_operator.py b/sdap_ingest_manager/config_operator.py
index fa2771a..97699b9 100644
--- a/sdap_ingest_manager/config_operator.py
+++ b/sdap_ingest_manager/config_operator.py
@@ -1,6 +1,5 @@
import argparse
-from sdap_ingest_manager.config import RemoteGitConfig, LocalDirConfig, ConfigMap
-
+from sdap_ingest_manager.config import RemoteGitConfig, LocalDirConfig, K8ConfigMap
def main():
@@ -16,6 +15,8 @@ def main():
parser.add_argument("-n", "--namespace", help="kubernetes namespace where the configuration will be deployed", required=True)
parser.add_argument("-cm", "--config-map", help="configmap name in kubernetes", required=True)
+ parser.add_argument("-u", "--updated-continuously", nargs='?', const=True, default=False,
+ help="k8 configMap is updated as soon as a syntactically correct configuration file is updated")
options = parser.parse_args()
@@ -24,10 +25,11 @@ def main():
else:
config = RemoteGitConfig(options.git_url, branch=options.git_branch, token=options.git_token)
- config_map = ConfigMap(options.config_map, options.namespace, config)
+ config_map = K8ConfigMap(options.config_map, options.namespace, config)
config_map.publish()
- config.when_updated(config_map.publish)
+ if options.updated_continuously:
+ config.when_updated(config_map.publish)
if __name__ == "__main__":
diff --git a/setup.py b/setup.py
index 75fdd72..7bd9810 100644
--- a/setup.py
+++ b/setup.py
@@ -42,9 +42,6 @@ setuptools.setup(
long_description_content_type="text/markdown",
url="https://github.com/tloubrieu-jpl/incubator-sdap-nexus-ingestion-manager",
packages=setuptools.find_packages(),
- scripts=['bin/run_collections',
- 'bin/run_single_collection',
- 'bin/run_granules'],
classifiers=[
"Programming Language :: Python :: 3",
"Operating System :: OS Independent",
@@ -53,7 +50,12 @@ setuptools.setup(
python_requires='>=3.6',
include_package_data=True,
data_files=[('.sdap_ingest_manager/resources/', ['sdap_ingest_manager/resources/dataset_config_template.yml'])],
- install_requires=pip_requirements
+ install_requires=pip_requirements,
+ entry_points={
+ 'config-operator': ['summary=sdap_ingest_manager.config_operator:main'],
+ 'collection-ingester': ['summary=sdap_ingest_manager.service:main'],
+ },
+
)
post_install_message()
diff --git a/tests/resources/data/collections.yml b/tests/resources/data/collections.yml
index 18226ba..42d2fbc 100644
--- a/tests/resources/data/collections.yml
+++ b/tests/resources/data/collections.yml
@@ -1,7 +1,7 @@
avhrr-oi-analysed-sst:
path: resources/history_manager/data/avhrr_oi/*.nc
variable: analysed_sst
- priority: 8
+ priority: 9
avhrr-oi-analysed-sst2:
path: resources/history_manager/data/avhrr_oi/*.nc
diff --git a/tests/resources/data/dataset_config_file_ok.yml b/tests/resources/data/dataset_config_file_ok.yml
index a000293..6ff0c47 100644
--- a/tests/resources/data/dataset_config_file_ok.yml
+++ b/tests/resources/data/dataset_config_file_ok.yml
@@ -7,7 +7,7 @@ ningester:
dimensions:
- lat
- lon
-
+ zobi;
---
# Tile processors configuration
ningester:
[incubator-sdap-ingester] 02/08: add config_operator working on
local dir or remote git repo
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 5fa6fb4060dc6a9252d2d840ed5f3dbbe91ffe7b
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Wed Jun 3 16:54:30 2020 -0700
add config_operator working on local dir or remote git repo
---
.gitignore | 4 +-
sdap_ingest_manager/config/ConfigMap.py | 69 +++++++++++++++++----------
sdap_ingest_manager/config/LocalDirConfig.py | 49 +++++++++++++++++++
sdap_ingest_manager/config/RemoteGitConfig.py | 62 ++++++++++++++++++++++++
sdap_ingest_manager/config/__init__.py | 3 ++
sdap_ingest_manager/config/exceptions.py | 4 ++
sdap_ingest_manager/config_operator.py | 35 ++++++++++++++
tests/config/test_ConfigMap.py | 16 ++-----
tests/resources/data/collections.yml | 4 +-
9 files changed, 206 insertions(+), 40 deletions(-)
diff --git a/.gitignore b/.gitignore
index b5cf7f0..0a97257 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,4 +7,6 @@ venv
__pycache__/
dist/
build/
-*.DS_Store
\ No newline at end of file
+*.DS_Store
+.eggs
+temp/
diff --git a/sdap_ingest_manager/config/ConfigMap.py b/sdap_ingest_manager/config/ConfigMap.py
index c5eb244..4143980 100644
--- a/sdap_ingest_manager/config/ConfigMap.py
+++ b/sdap_ingest_manager/config/ConfigMap.py
@@ -2,22 +2,25 @@ import logging
from kubernetes import client, config
from kubernetes.client.rest import ApiException
+from sdap_ingest_manager.config.exceptions import UnreadableFileException
+
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
+
class ConfigMap:
- def __init__(self, configmap_name, namespace, ingestion_order_store, output_collection='collections.yml'):
- self._ingestion_order_store = ingestion_order_store
+ def __init__(self, configmap_name, namespace, git_remote_config):
+ self._git_remote_config = git_remote_config
self._namespace = namespace
self._configmap_name = configmap_name
- self._output_collection = output_collection
+
config.load_kube_config()
configuration = client.Configuration()
- self._api_instance = client.CoreV1Api(client.ApiClient(configuration))
-
+ self._api_instance = client.ApiClient(configuration)
+ self._api_core_v1_instance = client.CoreV1Api(self._api_instance)
def __del__(self):
- pass
+ self._api_instance.close()
def _create_configmap_object(self):
@@ -25,9 +28,14 @@ class ConfigMap:
name=self._configmap_name ,
namespace=self._namespace,
)
-
- data = {self._output_collection:self._ingestion_order_store.get_content()}
+ data = {}
+ for f in self._git_remote_config.get_files():
+ try:
+ data[f] = self._git_remote_config.get_file_content(f)
+ except UnreadableFileException as e:
+ logger.error(f'file {f} cannot be read, ignored', e)
+
configmap = client.V1ConfigMap(
api_version="v1",
kind="ConfigMap",
@@ -37,8 +45,11 @@ class ConfigMap:
return configmap
def _get_deployed_config(self):
+ """
+ This method does not work in my test, the list of config available is not up to date
+ """
try:
- api_list_response = self._api_instance.list_namespaced_config_map(self._namespace)
+ api_list_response = self._api_core_v1_instance.list_namespaced_config_map(self._namespace)
config_keys = set()
for item in api_list_response.items:
config_keys = config_keys.union(item.data.keys())
@@ -47,24 +58,34 @@ class ConfigMap:
finally:
return config_keys
- def publish(self):
+ def _replace(self):
try:
+ logger.info(f'replace configMap entry {self._configmap_name}')
+ api_response = self._api_core_v1_instance.replace_namespaced_config_map(
+ name=self._configmap_name,
+ namespace=self._namespace,
+ body=self._create_configmap_object()
+ )
+ logger.info(api_response)
+ except ApiException as e:
+ raise e
- if self._output_collection in self._get_deployed_config():
- logger.info(f'replace configMap entry {self._output_collection}')
- api_response = self._api_instance.replace_namespaced_config_map(
- name=self._output_collection,
- namespace=self._namespace,
- body=self._create_configmap_object()
- )
- else:
- logger.info(f'create configMap entry {self._output_collection}')
- api_response = self._api_instance.create_namespaced_config_map(
- namespace=self._namespace,
- body=self._create_configmap_object()
- )
+ def _create(self):
+ try:
+ logger.info(f'create configMap entry {self._configmap_name}')
+ api_response = self._api_core_v1_instance.create_namespaced_config_map(
+ namespace=self._namespace,
+ body=self._create_configmap_object()
+ )
logger.info(api_response)
except ApiException as e:
- logger.error("Exception when calling Kubernetes CoreV1Api %s\n" % e)
+ raise e
+
+ def publish(self):
+ try:
+ self._create()
+ except ApiException as e:
+ logger.error("Exception when calling Kubernetes CoreV1Api ,create failed, try to replace %s\n" % e)
+ self._replace()
diff --git a/sdap_ingest_manager/config/LocalDirConfig.py b/sdap_ingest_manager/config/LocalDirConfig.py
new file mode 100644
index 0000000..d58f387
--- /dev/null
+++ b/sdap_ingest_manager/config/LocalDirConfig.py
@@ -0,0 +1,49 @@
+import os
+import time
+import logging
+
+from sdap_ingest_manager.config.exceptions import UnreadableFileException
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1
+
+
+class LocalDirConfig:
+ def __init__(self, local_dir):
+ self._local_dir = local_dir
+ self._latest_update = self._get_latest_update()
+
+ def get_files(self):
+ files = []
+ for f in os.listdir(self._local_dir):
+ if os.path.isfile(os.path.join(self._local_dir, f)) \
+ and 'README' not in f \
+ and not f.startswith('.'):
+ files.append(f)
+
+ return files
+
+ def get_file_content(self, file_name):
+ logger.info(f'read configuration file {file_name}')
+ try:
+ with open(os.path.join(self._local_dir, file_name)) as f:
+ return f.read()
+ except UnicodeDecodeError as e:
+ raise UnreadableFileException(e)
+
+ def _get_latest_update(self):
+ return time.ctime(max(os.path.getmtime(root) for root,_,_ in os.walk(self._local_dir)))
+
+ def when_updated(self, callback):
+ while True:
+ time.sleep(LISTEN_FOR_UPDATE_INTERVAL_SECONDS)
+ latest_update = self._get_latest_update()
+ if latest_update > self._latest_update:
+ logger.info("local config dir has been updated")
+ callback()
+ self._latest_update = latest_update
+ else:
+ logger.debug("local config dir has not been updated")
+
diff --git a/sdap_ingest_manager/config/RemoteGitConfig.py b/sdap_ingest_manager/config/RemoteGitConfig.py
new file mode 100644
index 0000000..a344246
--- /dev/null
+++ b/sdap_ingest_manager/config/RemoteGitConfig.py
@@ -0,0 +1,62 @@
+import logging
+import os
+import sys
+import time
+from git import Repo, Remote
+from sdap_ingest_manager.config import LocalDirConfig
+
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 5
+
+class RemoteGitConfig(LocalDirConfig):
+ def __init__(self, git_url,
+ git_branch='master',
+ git_token=None
+ ):
+ """
+
+ :param git_url:
+ :param git_branch:
+ :param git_token:
+ """
+ self._git_url = git_url if git_url.endswith(".git") else git_url + '.git'
+ self._git_branch = git_branch
+ self._git_token = git_token
+ local_dir = os.path.join(sys.prefix, 'sdap', 'conf')
+ super().__init__(local_dir)
+ self._repo = None
+ self._init_local_config_repo()
+ self._latest_commit_key = self._repo.head.commit.hexsha
+
+ def _pull_remote(self):
+ o = self._repo.remotes.origin
+ res = o.pull()
+ return res[0].commit.hexsha # return the latest commit key
+
+ def _init_local_config_repo(self):
+ self._repo = Repo.init(self._local_dir)
+ if len(self._repo.remotes) == 0 or 'origin' not in [r.name for r in self._repo.remotes]:
+ self._repo.create_remote('origin', self._git_url)
+ self._repo.git.fetch()
+ self._repo.git.checkout(self._git_branch)
+
+ def when_updated(self, callback):
+
+ while True:
+ time.sleep(LISTEN_FOR_UPDATE_INTERVAL_SECONDS)
+ remote_commit_key = self._pull_remote()
+ if remote_commit_key != self._latest_commit_key:
+ logger.info("remote git repository has been updated")
+ callback()
+ self._latest_commit_key = remote_commit_key
+ else:
+ logger.debug("remote git repository has not been updated")
+
+
+
+
+
+
diff --git a/sdap_ingest_manager/config/__init__.py b/sdap_ingest_manager/config/__init__.py
index 27a89b8..852920f 100644
--- a/sdap_ingest_manager/config/__init__.py
+++ b/sdap_ingest_manager/config/__init__.py
@@ -1 +1,4 @@
from sdap_ingest_manager.config.LocalConfiguration import LocalConfiguration
+from sdap_ingest_manager.config.ConfigMap import ConfigMap
+from sdap_ingest_manager.confg.LocalDirConfig import LocalDirConfig
+from sdpa_ingest_manager.config.RemoteGitConfig import RemoteGitConfig
\ No newline at end of file
diff --git a/sdap_ingest_manager/config/exceptions.py b/sdap_ingest_manager/config/exceptions.py
new file mode 100644
index 0000000..c06b881
--- /dev/null
+++ b/sdap_ingest_manager/config/exceptions.py
@@ -0,0 +1,4 @@
+
+
+class UnreadableFileException(Exception):
+ pass
\ No newline at end of file
diff --git a/sdap_ingest_manager/config_operator.py b/sdap_ingest_manager/config_operator.py
new file mode 100644
index 0000000..fa2771a
--- /dev/null
+++ b/sdap_ingest_manager/config_operator.py
@@ -0,0 +1,35 @@
+import argparse
+from sdap_ingest_manager.config import RemoteGitConfig, LocalDirConfig, ConfigMap
+
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Run git configuration synchronization operator, work on local-dir or git-url")
+ input_group = parser.add_mutually_exclusive_group(required=True)
+ input_group.add_argument("-l", "--local-dir",
+ help="local directory where the configuration files are")
+ input_group.add_argument("-gu", "--git-url",
+ help="git repository from which the configuration files are pulled/saved")
+ parser.add_argument("-gb", "--git-branch", help="git branch from which the configuration files are pulled/saved",
+ default="master")
+ parser.add_argument("-gt", "--git-token", help="git personal access token used to access the repository")
+
+ parser.add_argument("-n", "--namespace", help="kubernetes namespace where the configuration will be deployed", required=True)
+ parser.add_argument("-cm", "--config-map", help="configmap name in kubernetes", required=True)
+
+ options = parser.parse_args()
+
+ if options.local_dir:
+ config = LocalDirConfig(options.local_dir)
+ else:
+ config = RemoteGitConfig(options.git_url, branch=options.git_branch, token=options.git_token)
+
+ config_map = ConfigMap(options.config_map, options.namespace, config)
+ config_map.publish()
+
+ config.when_updated(config_map.publish)
+
+
+if __name__ == "__main__":
+ main()
+
diff --git a/tests/config/test_ConfigMap.py b/tests/config/test_ConfigMap.py
index a536202..2518b9a 100644
--- a/tests/config/test_ConfigMap.py
+++ b/tests/config/test_ConfigMap.py
@@ -5,25 +5,15 @@ from flask import Flask
from flask_restplus import Api
from sdap_ingest_manager.config.ConfigMap import ConfigMap
-from sdap_ingest_manager.ingestion_order_store.FileIngestionOrderStore import FileIngestionOrderStore
-from sdap_ingest_manager.ingestion_order_store.templates import Templates
+from sdap_ingest_manager.config.RemoteGitConfig import RemoteGitConfig
-flask_app = Flask(__name__)
-app = Api(app=flask_app)
-templates = Templates(app)
class ConfigMapTest(unittest.TestCase):
def test_createconfigmap(self):
- test_ingestion_order_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
- '..',
- 'resources',
- 'data',
- 'collections.yml')
- file_ingestion_order_store = FileIngestionOrderStore(path=test_ingestion_order_file,
- order_template=templates.order_template)
+ remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config")
- config_map = ConfigMap('collections.yml', 'sdap', file_ingestion_order_store)
+ config_map = ConfigMap('collection-ingester', 'sdap', remote_git_config)
config_map.publish()
diff --git a/tests/resources/data/collections.yml b/tests/resources/data/collections.yml
index 07e795b..18226ba 100644
--- a/tests/resources/data/collections.yml
+++ b/tests/resources/data/collections.yml
@@ -1,9 +1,9 @@
avhrr-oi-analysed-sst:
path: resources/history_manager/data/avhrr_oi/*.nc
variable: analysed_sst
- priority: 2
+ priority: 8
avhrr-oi-analysed-sst2:
path: resources/history_manager/data/avhrr_oi/*.nc
variable: analysed_sst
- priority: 1
\ No newline at end of file
+ priority: 1
[incubator-sdap-ingester] 01/08: initiate a configMap object with
unit test (with bug)
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 5520d27582029858ddc3e38725616971fd1c04e9
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Wed Jun 3 11:32:04 2020 -0700
initiate a configMap object with unit test (with bug)
---
requirements.txt | 1 +
sdap_ingest_manager/config/ConfigMap.py | 70 ++++++++++++++++++++++
.../GitIngestionOrderStore.py | 2 +
.../ingestion_order_store/IngestionOrderStore.py | 3 +
tests/config/__init__.py | 0
tests/config/test_ConfigMap.py | 32 ++++++++++
6 files changed, 108 insertions(+)
diff --git a/requirements.txt b/requirements.txt
index f4e39d0..996a27e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ Flask==1.1.2
flask-restplus==0.13.0
Werkzeug==0.16.0
pika==1.1.0
+kubernetes==11.0
\ No newline at end of file
diff --git a/sdap_ingest_manager/config/ConfigMap.py b/sdap_ingest_manager/config/ConfigMap.py
new file mode 100644
index 0000000..c5eb244
--- /dev/null
+++ b/sdap_ingest_manager/config/ConfigMap.py
@@ -0,0 +1,70 @@
+import logging
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class ConfigMap:
+ def __init__(self, configmap_name, namespace, ingestion_order_store, output_collection='collections.yml'):
+ self._ingestion_order_store = ingestion_order_store
+ self._namespace = namespace
+ self._configmap_name = configmap_name
+ self._output_collection = output_collection
+ config.load_kube_config()
+ configuration = client.Configuration()
+ self._api_instance = client.CoreV1Api(client.ApiClient(configuration))
+
+
+ def __del__(self):
+ pass
+
+ def _create_configmap_object(self):
+
+ metadata = client.V1ObjectMeta(
+ name=self._configmap_name ,
+ namespace=self._namespace,
+ )
+
+ data = {self._output_collection:self._ingestion_order_store.get_content()}
+
+ configmap = client.V1ConfigMap(
+ api_version="v1",
+ kind="ConfigMap",
+ data=data,
+ metadata=metadata
+ )
+ return configmap
+
+ def _get_deployed_config(self):
+ try:
+ api_list_response = self._api_instance.list_namespaced_config_map(self._namespace)
+ config_keys = set()
+ for item in api_list_response.items:
+ config_keys = config_keys.union(item.data.keys())
+ except ApiException as e:
+ logger.error("Exception when calling Kubernetes CoreV1Api %s\n" % e)
+ finally:
+ return config_keys
+
+ def publish(self):
+ try:
+
+ if self._output_collection in self._get_deployed_config():
+ logger.info(f'replace configMap entry {self._output_collection}')
+ api_response = self._api_instance.replace_namespaced_config_map(
+ name=self._output_collection,
+ namespace=self._namespace,
+ body=self._create_configmap_object()
+ )
+ else:
+ logger.info(f'create configMap entry {self._output_collection}')
+ api_response = self._api_instance.create_namespaced_config_map(
+ namespace=self._namespace,
+ body=self._create_configmap_object()
+ )
+ logger.info(api_response)
+
+ except ApiException as e:
+ logger.error("Exception when calling Kubernetes CoreV1Api %s\n" % e)
+
diff --git a/sdap_ingest_manager/ingestion_order_store/GitIngestionOrderStore.py b/sdap_ingest_manager/ingestion_order_store/GitIngestionOrderStore.py
index 349eeb2..1020679 100644
--- a/sdap_ingest_manager/ingestion_order_store/GitIngestionOrderStore.py
+++ b/sdap_ingest_manager/ingestion_order_store/GitIngestionOrderStore.py
@@ -2,6 +2,7 @@ import logging
import os
import sys
+
from git import Repo, Remote
from sdap_ingest_manager.ingestion_order_store.IngestionOrderStore import IngestionOrderStore
@@ -54,3 +55,4 @@ class GitIngestionOrderStore(IngestionOrderStore):
self._repo.create_remote('origin', self._git_url)
self._repo.git.fetch()
self._repo.git.checkout(self._git_branch)
+
diff --git a/sdap_ingest_manager/ingestion_order_store/IngestionOrderStore.py b/sdap_ingest_manager/ingestion_order_store/IngestionOrderStore.py
index c791fe3..89af10d 100644
--- a/sdap_ingest_manager/ingestion_order_store/IngestionOrderStore.py
+++ b/sdap_ingest_manager/ingestion_order_store/IngestionOrderStore.py
@@ -31,3 +31,6 @@ class IngestionOrderStore:
except FileNotFoundError:
logger.error(f"no collection configuration found at {self._ingestion_orders}")
+
+ def get_content(self):
+ return yaml.dump(self._ingestion_orders)
\ No newline at end of file
diff --git a/tests/config/__init__.py b/tests/config/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/config/test_ConfigMap.py b/tests/config/test_ConfigMap.py
new file mode 100644
index 0000000..a536202
--- /dev/null
+++ b/tests/config/test_ConfigMap.py
@@ -0,0 +1,32 @@
+import unittest
+import os
+
+from flask import Flask
+from flask_restplus import Api
+
+from sdap_ingest_manager.config.ConfigMap import ConfigMap
+from sdap_ingest_manager.ingestion_order_store.FileIngestionOrderStore import FileIngestionOrderStore
+from sdap_ingest_manager.ingestion_order_store.templates import Templates
+
+flask_app = Flask(__name__)
+app = Api(app=flask_app)
+templates = Templates(app)
+
+class ConfigMapTest(unittest.TestCase):
+ def test_createconfigmap(self):
+
+ test_ingestion_order_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ '..',
+ 'resources',
+ 'data',
+ 'collections.yml')
+ file_ingestion_order_store = FileIngestionOrderStore(path=test_ingestion_order_file,
+ order_template=templates.order_template)
+
+ config_map = ConfigMap('collections.yml', 'sdap', file_ingestion_order_store)
+ config_map.publish()
+
+
+
+if __name__ == '__main__':
+ unittest.main()
[incubator-sdap-ingester] 04/08: start to implement config-operator
containers
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit b8281218bcbb67df6429eb19766aa023a14158dc
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Thu Jun 4 16:18:32 2020 -0700
start to implement config-operator containers
---
.github/workflows/github-dev-release.yml | 6 +++
README.md | 56 ++++++++--------------
.../docker/{ => collection-ingester}/Dockerfile | 0
containers/docker/config-operator/Dockerfile | 10 ++++
containers/kubernetes/config-operator.yml | 39 +++++++++++++++
sdap_ingest_manager/__init__.py | 1 +
sdap_ingest_manager/config/K8ConfigMap.py | 2 +
sdap_ingest_manager/config/LocalDirConfig.py | 2 +-
setup.py | 29 ++---------
tests/resources/data/dataset_config_file_ok.yml | 1 -
10 files changed, 85 insertions(+), 61 deletions(-)
diff --git a/.github/workflows/github-dev-release.yml b/.github/workflows/github-dev-release.yml
index ae6e030..9f9b555 100644
--- a/.github/workflows/github-dev-release.yml
+++ b/.github/workflows/github-dev-release.yml
@@ -62,5 +62,11 @@ jobs:
run: |
pip install pds-github-util
python-snapshot-release --token ${{ secrets.GITHUB_TOKEN }}
+ - name: Publish the Python distribution to PyPI
+ uses: pypa/gh-action-pypi-publish@master
+ with:
+ user: ${{ secrets.pypi_username }}
+ password: ${{ secrets.pypi_password }}
+ repository_url: https://test.pypi.org/legacy/
diff --git a/README.md b/README.md
index 1dbdf70..d653212 100644
--- a/README.md
+++ b/README.md
@@ -125,9 +125,27 @@ Deploy a local rabbitmq service, for example with docker.
docker run -d --hostname localhost -p 5672:5672 --name rabbitmq rabbitmq:3
+
### Launch the service
- python sdap_ingest_manager/service.py --local-ingestion-orders=tests/resources/data/collections.yml --history-path=/tmp
+#### The config operator:
+
+This component helps to import a configuration directory on local file system or on a git repository as a configMap in kubernetes.
+This makes the configuration easily accessible to all the nodes of the cluster whereas the configuration stays in a single place.
+The configurations can be updated while the service is running (-u). The configuration updates will be published to kubernetes pods by patching the existing configurations.
+
+ config-operator -h
+ config-operator -l tests/resources/data -n sdap -cm collection-ingester-config
+ config-operator --git-url=https://github.com/tloubrieu-jpl/sdap-ingester-config --namespace=sdap --config-map=collection-ingester-config
+
+#### The collection ingestion service
+
+The service reads the collection configuration and submit granule ingestion messages to the message broker (rabbitmq).
+For each collection, 2 ingestion priority levels are proposed: the nominal priority, the priority for forward processing (newer files), usually higher.
+An history of the ingested granules is managed so that the ingestion can stop and re-start anytime.
+
+ collection-ingester -h
+ collection-ingester --local-ingestion-orders=tests/resources/data/collections.yml --history-path=/tmp
### Test and create the package
@@ -149,47 +167,15 @@ The release will be automatically pushed to pypi though github action.
## Docker
-(development version)
-
- cd containers/docker
- docker build --no-cache --tag tloubrieu/sdap-ingest-manager:latest .
- docker run -it --name sdap-ingest-manager -v sdap_ingest_config:/usr/local/.sdap_ingest_manager tloubrieu/sdap-ingest-manager:latest
- docker volume inspect sdap_ingest_config
-
-You can see the configuration files in the directory of the named volume (for example /var/lib/docker/volumes/sdap_ingest_config/_data).
-
-Note on macos, to access this directory, you need to go inside the Virtual Machine which runs docker service. To update the configuration on macos:
-
- docker run --rm -it -v /:/vm-root alpine:edge /bin/bash
- cd /vm-root/var/lib/docker/volumes/sdap_ingest_config/_data
- cp sdap_ingest_manager.ini.default sdap_ingest_manager.ini
- vi sdap_ingest_manager.ini
-
+ docker build . -f containers/docker/config-operator/Dockerfile --no-cache --tag tloubrieu/sdap-ingest-manager:latest
+
To publish the docker image on dockerhub do (step necessary for kubernetes deployment):
docker login
docker push tloubrieu/sdap-ingest-manager:latest
## Kubernetes
-
-### Create the configMap for your deployment
-
-Prepare a configMap from existing native config files:
-
- kubectl create configmap collection-ingester-config --from-file=venv/.sdap_ingest_manager -n sdap
-
-#### Optionally you can update the configMap manually if the one you started from is not what you needed:
- kubectl get configmap collection-ingester-config -o yaml -n sdap > containers/kubernetes/sdap_ingester_config.yml
-
-Manually edit the yml file to only keep the configuration which is specific to the deployment (if different from the current one)
-
-Replace the configmap:
-
- kubectl delete configmap collection-ingester-config -n sdap
- kubectl apply -f containers/kubernetes/sdap_ingester_config.yml -n sdap
-
-
### Launch the service
kubectl apply -f containers/kubernetes/job.yml -n sdap
diff --git a/containers/docker/Dockerfile b/containers/docker/collection-ingester/Dockerfile
similarity index 100%
rename from containers/docker/Dockerfile
rename to containers/docker/collection-ingester/Dockerfile
diff --git a/containers/docker/config-operator/Dockerfile b/containers/docker/config-operator/Dockerfile
new file mode 100644
index 0000000..81918b8
--- /dev/null
+++ b/containers/docker/config-operator/Dockerfile
@@ -0,0 +1,10 @@
+FROM python:3
+
+COPY /sdap_ingest_manager /sdap_ingest_manager/sdap_ingest_manager
+COPY /setup.py /sdap_ingest_manager/setup.py
+COPY /requirements.txt /sdap_ingest_manager/requirements.txt
+COPY /README.md /sdap_ingest_manager/README.md
+
+RUN cd /sdap_ingest_manager && pip install .
+
+CMD bash
diff --git a/containers/kubernetes/config-operator.yml b/containers/kubernetes/config-operator.yml
new file mode 100644
index 0000000..1d8bc16
--- /dev/null
+++ b/containers/kubernetes/config-operator.yml
@@ -0,0 +1,39 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: collection-ingester
+spec:
+ template:
+ spec:
+ containers:
+ - name: collections-ingester
+ image: tloubrieu/sdap-ingest-manager:latest
+ imagePullPolicy: IfNotPresent
+ command: ["run_collections", "--config=/opt/sdap_ingester_config/"]
+ volumeMounts:
+ - name: config-vol
+ mountPath: /opt/sdap_ingester_config/
+ - name: data-volume-for-collection-ingester
+ mountPath: /data
+ readOnly: true
+ volumes:
+ - name: config-vol
+ configMap:
+ name: collection-ingester-config
+ - name: data-volume-for-collection-ingester
+ #hostPath:
+ # path: /Users/loubrieu/PycharmProjects/sdap_ingest_manager/sdap_ingest_manager/ingestion_order_executor/history_manager/data
+ # type: Directory
+ persistentVolumeClaim:
+ claimName: data-volume-claim
+
+ restartPolicy: Never
+ backoffLimit: 4
+
+---
+
+
+
+
+
+
diff --git a/sdap_ingest_manager/__init__.py b/sdap_ingest_manager/__init__.py
index e69de29..51d7666 100644
--- a/sdap_ingest_manager/__init__.py
+++ b/sdap_ingest_manager/__init__.py
@@ -0,0 +1 @@
+__version__='1.0.0.dev0'
\ No newline at end of file
diff --git a/sdap_ingest_manager/config/K8ConfigMap.py b/sdap_ingest_manager/config/K8ConfigMap.py
index 156bb19..e9afe55 100644
--- a/sdap_ingest_manager/config/K8ConfigMap.py
+++ b/sdap_ingest_manager/config/K8ConfigMap.py
@@ -59,6 +59,8 @@ class K8ConfigMap:
return config_keys
def _patch(self):
+ """ replaces files available in the config but does not delete
+ what is not available (e.g. which has not been parsed)"""
try:
logger.info(f'replace configMap entry {self._configmap_name}')
api_response = self._api_core_v1_instance.patch_namespaced_config_map(
diff --git a/sdap_ingest_manager/config/LocalDirConfig.py b/sdap_ingest_manager/config/LocalDirConfig.py
index 515ee22..6d8a7cf 100644
--- a/sdap_ingest_manager/config/LocalDirConfig.py
+++ b/sdap_ingest_manager/config/LocalDirConfig.py
@@ -27,7 +27,7 @@ class LocalDirConfig:
return files
def _test_read_yaml(self, file_name):
- """ check yaml syntax raiseyaml.parser.ParserError is it doesn't"""
+ """ check yaml syntax raise yaml.parser.ParserError is it doesn't"""
with open(os.path.join(self._local_dir, file_name), 'r') as f:
docs = yaml.load_all(f, Loader=yaml.FullLoader)
for doc in docs:
diff --git a/setup.py b/setup.py
index 7bd9810..f85d6a7 100644
--- a/setup.py
+++ b/setup.py
@@ -6,22 +6,6 @@ import re
PACKAGE_NAME = "sdap_ingest_manager"
-
-def post_install_message():
- try:
- from tabulate import tabulate
- except ImportError:
- subprocess.call([sys.executable, "-m", "pip", "install", 'tabulate'])
- finally:
- from tabulate import tabulate
-
- path_to_configuration_files = os.path.join(sys.prefix, f".{PACKAGE_NAME}")
- message = f"Now, create configuration files in \n" \
- f"***{path_to_configuration_files}*** \n" \
- f" Use templates and examples provided there"
- print(tabulate([[message]]))
-
-
with open("./sdap_ingest_manager/__init__.py") as fi:
result = re.search(r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fi.read())
version = result.group(1)
@@ -29,8 +13,8 @@ version = result.group(1)
with open("README.md", "r") as fh:
long_description = fh.read()
-with open('requirements.txt') as f:
- pip_requirements = f.readlines()
+with open('requirements.txt', 'r') as f:
+ requirements = f.readlines()
setuptools.setup(
name=PACKAGE_NAME,
@@ -50,12 +34,9 @@ setuptools.setup(
python_requires='>=3.6',
include_package_data=True,
data_files=[('.sdap_ingest_manager/resources/', ['sdap_ingest_manager/resources/dataset_config_template.yml'])],
- install_requires=pip_requirements,
+ install_requires=requirements,
entry_points={
- 'config-operator': ['summary=sdap_ingest_manager.config_operator:main'],
- 'collection-ingester': ['summary=sdap_ingest_manager.service:main'],
+ 'console_scripts': ['config-operator=sdap_ingest_manager.config_operator:main',
+ 'collection-ingester=sdap_ingest_manager.service:main']
},
-
)
-
-post_install_message()
diff --git a/tests/resources/data/dataset_config_file_ok.yml b/tests/resources/data/dataset_config_file_ok.yml
index 6ff0c47..66bb883 100644
--- a/tests/resources/data/dataset_config_file_ok.yml
+++ b/tests/resources/data/dataset_config_file_ok.yml
@@ -7,7 +7,6 @@ ningester:
dimensions:
- lat
- lon
- zobi;
---
# Tile processors configuration
ningester:
[incubator-sdap-ingester] 06/08: integrates with kubernetes
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 3268385e005ebfcc52ba7645b151ce72c6f9dfa8
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Sun Jun 7 10:57:57 2020 -0700
integrates with kubernetes
---
config_operator/README.md | 2 +-
config_operator/config_operator/config_operator.py | 5 +++--
config_operator/config_operator/config_source/LocalDirConfig.py | 8 ++++++--
config_operator/config_operator/config_source/RemoteGitConfig.py | 8 +++++---
config_operator/config_operator/k8s/K8sConfigMap.py | 6 +++++-
.../containers/k8s/{deployment.yml => deployment-git-src.yml} | 2 ++
6 files changed, 22 insertions(+), 9 deletions(-)
diff --git a/config_operator/README.md b/config_operator/README.md
index 61889c4..371a604 100644
--- a/config_operator/README.md
+++ b/config_operator/README.md
@@ -35,4 +35,4 @@ To publish the docker image on dockerhub do (step necessary for kubernetes deplo
## Kubernetes
- kubectl apply -f containers/k8s/deployment.yml -n sdap
\ No newline at end of file
+ kubectl apply -f containers/k8s/deployment-git-src.yml -n sdap
\ No newline at end of file
diff --git a/config_operator/config_operator/config_operator.py b/config_operator/config_operator/config_operator.py
index b032384..6b512db 100644
--- a/config_operator/config_operator/config_operator.py
+++ b/config_operator/config_operator/config_operator.py
@@ -12,10 +12,11 @@ def main():
help="git repository from which the configuration files are pulled/saved")
parser.add_argument("-gb", "--git-branch", help="git branch from which the configuration files are pulled/saved",
default="master")
+ parser.add_argument("-gl", "--git-local", help="local git repository", required=False)
parser.add_argument("-gt", "--git-token", help="git personal access token used to access the repository")
-
parser.add_argument("-n", "--namespace", help="kubernetes namespace where the configuration will be deployed", required=True)
parser.add_argument("-cm", "--config-map", help="configmap name in kubernetes", required=True)
+
parser.add_argument("-u", "--updated-continuously", nargs='?', const=True, default=False,
help="k8 configMap is updated as soon as a syntactically correct configuration file is updated")
@@ -24,7 +25,7 @@ def main():
if options.local_dir:
config = LocalDirConfig(options.local_dir)
else:
- config = RemoteGitConfig(options.git_url, branch=options.git_branch, token=options.git_token)
+ config = RemoteGitConfig(options.git_url, branch=options.git_branch, token=options.git_token, local_dir=options.git_local)
config_map = K8sConfigMap(options.config_map, options.namespace, config)
config_map.publish()
diff --git a/config_operator/config_operator/config_source/LocalDirConfig.py b/config_operator/config_operator/config_source/LocalDirConfig.py
index 89c0e5a..f37e41b 100644
--- a/config_operator/config_operator/config_source/LocalDirConfig.py
+++ b/config_operator/config_operator/config_source/LocalDirConfig.py
@@ -49,13 +49,17 @@ class LocalDirConfig:
def _get_latest_update(self):
- return time.ctime(max(os.path.getmtime(root) for root,_,_ in os.walk(self._local_dir)))
+ m_times = [os.path.getmtime(root) for root, _, _ in os.walk(self._local_dir)]
+ if m_times:
+ return time.ctime(max(m_times))
+ else:
+ return None
def when_updated(self, callback):
while True:
time.sleep(LISTEN_FOR_UPDATE_INTERVAL_SECONDS)
latest_update = self._get_latest_update()
- if latest_update > self._latest_update:
+ if latest_update is None or (latest_update > self._latest_update):
logger.info("local config dir has been updated")
callback()
self._latest_update = latest_update
diff --git a/config_operator/config_operator/config_source/RemoteGitConfig.py b/config_operator/config_operator/config_source/RemoteGitConfig.py
index 24e614a..15c0f01 100644
--- a/config_operator/config_operator/config_source/RemoteGitConfig.py
+++ b/config_operator/config_operator/config_source/RemoteGitConfig.py
@@ -9,12 +9,13 @@ logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 5
-
+DEFAULT_LOCAL_REPO_DIR = os.path.join(sys.prefix, 'sdap', 'conf')
class RemoteGitConfig(LocalDirConfig):
def __init__(self, git_url,
branch='master',
- token=None
+ token=None,
+ local_dir=DEFAULT_LOCAL_REPO_DIR
):
"""
@@ -25,7 +26,8 @@ class RemoteGitConfig(LocalDirConfig):
self._git_url = git_url if git_url.endswith(".git") else git_url + '.git'
self._git_branch = branch
self._git_token = token
- local_dir = os.path.join(sys.prefix, 'sdap', 'conf')
+ if local_dir is None:
+ local_dir = DEFAULT_LOCAL_REPO_DIR
super().__init__(local_dir)
self._repo = None
self._init_local_config_repo()
diff --git a/config_operator/config_operator/k8s/K8sConfigMap.py b/config_operator/config_operator/k8s/K8sConfigMap.py
index b16b58c..f7784cb 100644
--- a/config_operator/config_operator/k8s/K8sConfigMap.py
+++ b/config_operator/config_operator/k8s/K8sConfigMap.py
@@ -1,3 +1,4 @@
+import os
import logging
from kubernetes import client, config
from kubernetes.client.rest import ApiException
@@ -14,7 +15,10 @@ class K8sConfigMap:
self._namespace = namespace
self._configmap_name = configmap_name
- config.load_kube_config()
+ if os.getenv('KUBERNETES_SERVICE_HOST'):
+ config.load_incluster_config()
+ else:
+ config.load_kube_config()
configuration = client.Configuration()
self._api_instance = client.ApiClient(configuration)
self._api_core_v1_instance = client.CoreV1Api(self._api_instance)
diff --git a/config_operator/containers/k8s/deployment.yml b/config_operator/containers/k8s/deployment-git-src.yml
similarity index 99%
rename from config_operator/containers/k8s/deployment.yml
rename to config_operator/containers/k8s/deployment-git-src.yml
index 1501a75..99b268d 100644
--- a/config_operator/containers/k8s/deployment.yml
+++ b/config_operator/containers/k8s/deployment-git-src.yml
@@ -19,3 +19,5 @@ spec:
image: tloubrieu/config-operator:latest
imagePullPolicy: IfNotPresent
command: ['config-operator', '--git-url', 'https://github.com/tloubrieu-jpl/sdap-ingester-config' , '--namespace', 'sdap', '--config-map', 'collection-ingester-conf', '-u']
+
+
[incubator-sdap-ingester] 07/08: run config operator as a k8s
operator
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 1fbfce557505b34286de17a082fde72511703508
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Mon Jun 8 17:27:42 2020 -0700
run config operator as a k8s operator
---
config_operator/README.md | 18 ++++-
config_operator/config_operator/config_operator.py | 39 -----------
.../config_operator/k8s/K8sConfigMap.py | 1 +
config_operator/config_operator/main.py | 32 +++++++++
config_operator/containers/docker/Dockerfile | 3 +-
.../containers/k8s/config-operator-crd.yml | 77 ++++++++++++++++++++++
config_operator/requirements.txt | 1 +
7 files changed, 130 insertions(+), 41 deletions(-)
diff --git a/config_operator/README.md b/config_operator/README.md
index 371a604..2e1ea0d 100644
--- a/config_operator/README.md
+++ b/config_operator/README.md
@@ -35,4 +35,20 @@ To publish the docker image on dockerhub do (step necessary for kubernetes deplo
## Kubernetes
- kubectl apply -f containers/k8s/deployment-git-src.yml -n sdap
\ No newline at end of file
+Deploy the gitbasedconfig operator:
+
+ kubectl apply -f containers/k8s/config-operator-crd.yml -n sdap
+
+Deploy the git custom resource which will be synchronize with a k8s configmap
+
+ kubectl apply -f containers/k8s/git-repo-test.yml -n sdap
+
+Check that the custom resource is deployed:
+
+ kubectl get gitbasedconfigs -n sdap
+
+Check that the configMap has been generated:
+
+ kubectl get configmaps -n sdap
+
+
\ No newline at end of file
diff --git a/config_operator/config_operator/config_operator.py b/config_operator/config_operator/config_operator.py
deleted file mode 100644
index 6b512db..0000000
--- a/config_operator/config_operator/config_operator.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import argparse
-from config_operator.config_source import RemoteGitConfig, LocalDirConfig
-from config_operator.k8s import K8sConfigMap
-
-
-def main():
- parser = argparse.ArgumentParser(description="Run git configuration synchronization operator, work on local-dir or git-url")
- input_group = parser.add_mutually_exclusive_group(required=True)
- input_group.add_argument("-l", "--local-dir",
- help="local directory where the configuration files are")
- input_group.add_argument("-gu", "--git-url",
- help="git repository from which the configuration files are pulled/saved")
- parser.add_argument("-gb", "--git-branch", help="git branch from which the configuration files are pulled/saved",
- default="master")
- parser.add_argument("-gl", "--git-local", help="local git repository", required=False)
- parser.add_argument("-gt", "--git-token", help="git personal access token used to access the repository")
- parser.add_argument("-n", "--namespace", help="kubernetes namespace where the configuration will be deployed", required=True)
- parser.add_argument("-cm", "--config-map", help="configmap name in kubernetes", required=True)
-
- parser.add_argument("-u", "--updated-continuously", nargs='?', const=True, default=False,
- help="k8 configMap is updated as soon as a syntactically correct configuration file is updated")
-
- options = parser.parse_args()
-
- if options.local_dir:
- config = LocalDirConfig(options.local_dir)
- else:
- config = RemoteGitConfig(options.git_url, branch=options.git_branch, token=options.git_token, local_dir=options.git_local)
-
- config_map = K8sConfigMap(options.config_map, options.namespace, config)
- config_map.publish()
-
- if options.updated_continuously:
- config.when_updated(config_map.publish)
-
-
-if __name__ == "__main__":
- main()
-
diff --git a/config_operator/config_operator/k8s/K8sConfigMap.py b/config_operator/config_operator/k8s/K8sConfigMap.py
index f7784cb..32ee844 100644
--- a/config_operator/config_operator/k8s/K8sConfigMap.py
+++ b/config_operator/config_operator/k8s/K8sConfigMap.py
@@ -22,6 +22,7 @@ class K8sConfigMap:
configuration = client.Configuration()
self._api_instance = client.ApiClient(configuration)
self._api_core_v1_instance = client.CoreV1Api(self._api_instance)
+ self.publish()
def __del__(self):
self._api_instance.close()
diff --git a/config_operator/config_operator/main.py b/config_operator/config_operator/main.py
new file mode 100644
index 0000000..3d6ad2e
--- /dev/null
+++ b/config_operator/config_operator/main.py
@@ -0,0 +1,32 @@
+import kopf
+from config_operator.config_source import RemoteGitConfig
+from config_operator.k8s import K8sConfigMap
+
+
+@kopf.on.create('sdap.apache.org', 'v1', 'git-repo-configs')
+def create_fn(body, spec, **kwargs):
+ # Get info from Git Repo Config object
+ name = body['metadata']['name']
+ namespace = body['metadata']['namespace']
+
+ if 'git-url' not in spec.keys():
+ raise kopf.HandlerFatalError(f"git-url must be set.")
+ if 'config-map' not in spec.keys():
+ raise kopf.HandlerFatalError(f"config-map must be set.")
+
+ git_url = spec['git-url']
+ config_map = spec['config-map']
+
+ _kargs = {}
+ for k in {'git-branch', 'git-token'}:
+ if k in spec.keys():
+ _kargs[k.split('-')[0]] = spec[k]
+
+ config = RemoteGitConfig(git_url, **_kargs)
+
+ config_map = K8sConfigMap(config_map, namespace, config)
+
+ config.when_updated(config_map.publish)
+
+ msg = f"configmap {config_map} created from git repo {git_url}"
+ return {'message': msg}
diff --git a/config_operator/containers/docker/Dockerfile b/config_operator/containers/docker/Dockerfile
index 4e82c98..4765d30 100644
--- a/config_operator/containers/docker/Dockerfile
+++ b/config_operator/containers/docker/Dockerfile
@@ -6,5 +6,6 @@ COPY /requirements.txt /config_operator/requirements.txt
COPY /README.md /config_operator/README.md
RUN cd /config_operator && pip install .
+COPY /config_operator/k8_config_operator.py /k8_config_operator.py
-CMD bash
+CMD ["kopf", "run", "/k8_config_operator.py", "--verbose"]
diff --git a/config_operator/containers/k8s/config-operator-crd.yml b/config_operator/containers/k8s/config-operator-crd.yml
new file mode 100644
index 0000000..ea9bbba
--- /dev/null
+++ b/config_operator/containers/k8s/config-operator-crd.yml
@@ -0,0 +1,77 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: gitbasedconfigs.sdap.apache.org
+spec:
+ group: sdap.apache.org
+ versions:
+ - name: v1
+ served: true
+ storage: true
+ scope: Namespaced
+ names:
+ plural: gitbasedconfigs
+ singular: gitbasedconfig
+ kind: gitBasedConfig
+ shortNames:
+ - gitcfg
+ validation:
+ openAPIV3Schema:
+ type: object
+ properties:
+ git-url:
+ type: string
+ git-branch:
+ type: string
+ git-token:
+ type: string
+ local-dir:
+ type: string
+ namespace:
+ type: string
+ config-map:
+ type: string
+
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: git-repo-config-operator
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: git-repo-config-operator
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cluster-admin
+subjects:
+ - kind: ServiceAccount
+ name: git-repo-config-operator
+ namespace: default
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: git-repo-config-operator
+spec:
+ selector:
+ matchLabels:
+ app: git-repo-config-operator
+ template:
+ metadata:
+ labels:
+ app: git-repo-config-operator
+ spec:
+ serviceAccountName: git-repo-config-operator
+ containers:
+ - image: tloubrieu/config-operator:latest
+ name: git-repo-config-operator
+ imagePullPolicy: IfNotPresent
\ No newline at end of file
diff --git a/config_operator/requirements.txt b/config_operator/requirements.txt
index 4365d3d..5d452e2 100644
--- a/config_operator/requirements.txt
+++ b/config_operator/requirements.txt
@@ -1,3 +1,4 @@
GitPython==3.1.2
kubernetes==11.0
+kopf==0.26