You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/06/10 20:39:09 UTC
[incubator-sdap-ingester] 08/08: apply change requests by pull
request #3 review
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 7a655f049c0191aed40352de9162f9d27f1793fb
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Mon Jun 8 19:48:44 2020 -0700
apply change requests by pull request #3 review
---
config_operator/README.md | 2 ++
.../config_source/LocalDirConfig.py | 18 +++++++++----
.../config_source/RemoteGitConfig.py | 30 +++++++++++++---------
.../config_operator/k8s/K8sConfigMap.py | 13 +++++++---
config_operator/config_operator/main.py | 14 +++++++---
config_operator/containers/docker/Dockerfile | 4 +--
.../containers/k8s/config-operator-crd.yml | 2 ++
.../tests/{config => config_source}/__init__.py | 0
config_operator/tests/{config => k8s}/__init__.py | 0
.../test_ConfigMap.py => k8s/test_K8sConfigMap.py} | 2 +-
10 files changed, 58 insertions(+), 27 deletions(-)
diff --git a/config_operator/README.md b/config_operator/README.md
index 2e1ea0d..2ff8072 100644
--- a/config_operator/README.md
+++ b/config_operator/README.md
@@ -51,4 +51,6 @@ Check that the configMap has been generated:
kubectl get configmaps -n sdap
+
+
\ No newline at end of file
diff --git a/config_operator/config_operator/config_source/LocalDirConfig.py b/config_operator/config_operator/config_source/LocalDirConfig.py
index f37e41b..cf95f42 100644
--- a/config_operator/config_operator/config_source/LocalDirConfig.py
+++ b/config_operator/config_operator/config_source/LocalDirConfig.py
@@ -2,6 +2,7 @@ import os
import time
import logging
import yaml
+from typing import Callable
from config_operator.config_source.exceptions import UnreadableFileException
@@ -14,10 +15,12 @@ LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1
class LocalDirConfig:
- def __init__(self, local_dir):
+ def __init__(self, local_dir: str,
+ update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS):
logger.info(f'create config on local dir {local_dir}')
self._local_dir = local_dir
self._latest_update = self._get_latest_update()
+ self._update_every_seconds=update_every_seconds
def get_files(self):
files = []
@@ -29,14 +32,14 @@ class LocalDirConfig:
return files
- def _test_read_yaml(self, file_name):
+ def _test_read_yaml(self, file_name: str):
""" check yaml syntax raise yaml.parser.ParserError is it doesn't"""
with open(os.path.join(self._local_dir, file_name), 'r') as f:
docs = yaml.load_all(f, Loader=yaml.FullLoader)
for doc in docs:
pass
- def get_file_content(self, file_name):
+ def get_file_content(self, file_name: str):
logger.info(f'read configuration file {file_name}')
try:
self._test_read_yaml(file_name)
@@ -55,9 +58,12 @@ class LocalDirConfig:
else:
return None
- def when_updated(self, callback):
+ def when_updated(self, callback: Callable[[], None]):
+ """
+ call function callback when the local directory is updated.
+ """
while True:
- time.sleep(LISTEN_FOR_UPDATE_INTERVAL_SECONDS)
+ time.sleep(self._update_every_seconds)
latest_update = self._get_latest_update()
if latest_update is None or (latest_update > self._latest_update):
logger.info("local config dir has been updated")
@@ -66,3 +72,5 @@ class LocalDirConfig:
else:
logger.debug("local config dir has not been updated")
+ return None
+
diff --git a/config_operator/config_operator/config_source/RemoteGitConfig.py b/config_operator/config_operator/config_source/RemoteGitConfig.py
index 15c0f01..17d8223 100644
--- a/config_operator/config_operator/config_source/RemoteGitConfig.py
+++ b/config_operator/config_operator/config_source/RemoteGitConfig.py
@@ -3,6 +3,7 @@ import os
import sys
import time
from git import Repo
+from typing import Callable
from .LocalDirConfig import LocalDirConfig
logging.basicConfig(level=logging.DEBUG)
@@ -11,11 +12,13 @@ logger = logging.getLogger(__name__)
LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 5
DEFAULT_LOCAL_REPO_DIR = os.path.join(sys.prefix, 'sdap', 'conf')
+
class RemoteGitConfig(LocalDirConfig):
- def __init__(self, git_url,
- branch='master',
- token=None,
- local_dir=DEFAULT_LOCAL_REPO_DIR
+ def __init__(self, git_url: str,
+ git_branch: str = 'master',
+ git_token: str = None,
+ update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS,
+ local_dir: str = DEFAULT_LOCAL_REPO_DIR
):
"""
@@ -24,11 +27,12 @@ class RemoteGitConfig(LocalDirConfig):
:param git_token:
"""
self._git_url = git_url if git_url.endswith(".git") else git_url + '.git'
- self._git_branch = branch
- self._git_token = token
+ self._git_branch = git_branch
+ self._git_token = git_token
if local_dir is None:
local_dir = DEFAULT_LOCAL_REPO_DIR
- super().__init__(local_dir)
+ self._update_every_seconds = update_every_seconds
+ super().__init__(local_dir, update_every_seconds=self._update_every_seconds)
self._repo = None
self._init_local_config_repo()
self._latest_commit_key = self._pull_remote()
@@ -45,12 +49,12 @@ class RemoteGitConfig(LocalDirConfig):
self._repo.git.fetch()
self._repo.git.checkout(self._git_branch)
-
-
- def when_updated(self, callback):
-
+ def when_updated(self, callback: Callable[[], None]):
+ """
+ call function callback when the remote git repository is updated.
+ """
while True:
- time.sleep(LISTEN_FOR_UPDATE_INTERVAL_SECONDS)
+ time.sleep(self._update_every_seconds)
remote_commit_key = self._pull_remote()
if remote_commit_key != self._latest_commit_key:
logger.info("remote git repository has been updated")
@@ -59,3 +63,5 @@ class RemoteGitConfig(LocalDirConfig):
else:
logger.debug("remote git repository has not been updated")
+ return None
+
diff --git a/config_operator/config_operator/k8s/K8sConfigMap.py b/config_operator/config_operator/k8s/K8sConfigMap.py
index 32ee844..e2a7a10 100644
--- a/config_operator/config_operator/k8s/K8sConfigMap.py
+++ b/config_operator/config_operator/k8s/K8sConfigMap.py
@@ -1,7 +1,9 @@
import os
import logging
from kubernetes import client, config
+from config_operator.config_source import LocalDirConfig, RemoteGitConfig
from kubernetes.client.rest import ApiException
+from typing import Union
from config_operator.config_source.exceptions import UnreadableFileException
@@ -10,11 +12,14 @@ logger = logging.getLogger(__name__)
class K8sConfigMap:
- def __init__(self, configmap_name, namespace, git_remote_config):
- self._git_remote_config = git_remote_config
+ def __init__(self, configmap_name: str,
+ namespace: str,
+ external_config: Union[LocalDirConfig, RemoteGitConfig]):
+ self._git_remote_config = external_config
self._namespace = namespace
self._configmap_name = configmap_name
+ # test is this runs inside kubernetes cluster
if os.getenv('KUBERNETES_SERVICE_HOST'):
config.load_incluster_config()
else:
@@ -93,6 +98,8 @@ class K8sConfigMap:
try:
self._create()
except ApiException as e:
- logger.error("Exception when calling Kubernetes CoreV1Api ,create failed, try to replace %s\n" % e)
+ logger.error("Exception when calling Kubernetes CoreV1Api ,create failed, try to patch %s\n" % e)
self._patch()
+ return None
+
diff --git a/config_operator/config_operator/main.py b/config_operator/config_operator/main.py
index 3d6ad2e..db4dbcb 100644
--- a/config_operator/config_operator/main.py
+++ b/config_operator/config_operator/main.py
@@ -1,9 +1,12 @@
+import logging
import kopf
from config_operator.config_source import RemoteGitConfig
from config_operator.k8s import K8sConfigMap
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
-@kopf.on.create('sdap.apache.org', 'v1', 'git-repo-configs')
+@kopf.on.create('sdap.apache.org', 'v1', 'gitbasedconfigs')
def create_fn(body, spec, **kwargs):
# Get info from Git Repo Config object
name = body['metadata']['name']
@@ -15,12 +18,15 @@ def create_fn(body, spec, **kwargs):
raise kopf.HandlerFatalError(f"config-map must be set.")
git_url = spec['git-url']
+ logger.info(f'git-url = {git_url}')
config_map = spec['config-map']
+ logger.info(f'config-map = {config_map}')
_kargs = {}
- for k in {'git-branch', 'git-token'}:
- if k in spec.keys():
- _kargs[k.split('-')[0]] = spec[k]
+ for k in {'git-branch', 'git-token', 'update-every-seconds'}:
+ if k in spec:
+ logger.info(f'{k} = {spec[k]}')
+ _kargs[k.replace('-', '_')] = spec[k]
config = RemoteGitConfig(git_url, **_kargs)
diff --git a/config_operator/containers/docker/Dockerfile b/config_operator/containers/docker/Dockerfile
index 4765d30..a161dae 100644
--- a/config_operator/containers/docker/Dockerfile
+++ b/config_operator/containers/docker/Dockerfile
@@ -6,6 +6,6 @@ COPY /requirements.txt /config_operator/requirements.txt
COPY /README.md /config_operator/README.md
RUN cd /config_operator && pip install .
-COPY /config_operator/k8_config_operator.py /k8_config_operator.py
+COPY /config_operator/main.py /main.py
-CMD ["kopf", "run", "/k8_config_operator.py", "--verbose"]
+CMD ["kopf", "run", "/main.py", "--verbose"]
diff --git a/config_operator/containers/k8s/config-operator-crd.yml b/config_operator/containers/k8s/config-operator-crd.yml
index ea9bbba..9f6d2ad 100644
--- a/config_operator/containers/k8s/config-operator-crd.yml
+++ b/config_operator/containers/k8s/config-operator-crd.yml
@@ -25,6 +25,8 @@ spec:
type: string
git-token:
type: string
+ update-every-seconds:
+ type: int
local-dir:
type: string
namespace:
diff --git a/config_operator/tests/config/__init__.py b/config_operator/tests/config_source/__init__.py
similarity index 100%
copy from config_operator/tests/config/__init__.py
copy to config_operator/tests/config_source/__init__.py
diff --git a/config_operator/tests/config/__init__.py b/config_operator/tests/k8s/__init__.py
similarity index 100%
rename from config_operator/tests/config/__init__.py
rename to config_operator/tests/k8s/__init__.py
diff --git a/config_operator/tests/config/test_ConfigMap.py b/config_operator/tests/k8s/test_K8sConfigMap.py
similarity index 95%
rename from config_operator/tests/config/test_ConfigMap.py
rename to config_operator/tests/k8s/test_K8sConfigMap.py
index 4d339e3..1660e69 100644
--- a/config_operator/tests/config/test_ConfigMap.py
+++ b/config_operator/tests/k8s/test_K8sConfigMap.py
@@ -5,7 +5,7 @@ from config_operator.k8s import K8sConfigMap
from config_operator.config_source import RemoteGitConfig, LocalDirConfig
-class ConfigMapTest(unittest.TestCase):
+class K8sConfigMapTest(unittest.TestCase):
def test_createconfigmapfromgit(self):
remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config")