You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/07/30 00:54:32 UTC
[incubator-sdap-ingester] branch dev updated: SDAP-247:
config-operator unit tests and support for git username/token (#6)
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/dev by this push:
new ecd5ecc SDAP-247: config-operator unit tests and support for git username/token (#6)
ecd5ecc is described below
commit ecd5ecc9804b6e7e213f443b3ec94f72f897a497
Author: thomas loubrieu <60...@users.noreply.github.com>
AuthorDate: Wed Jul 29 17:54:25 2020 -0700
SDAP-247: config-operator unit tests and support for git username/token (#6)
Co-authored-by: thomas loubrieu <th...@jpl.nasa.gov>
Co-authored-by: Eamon Ford <ea...@jpl.nasa.gov>
---
config_operator/README.md | 6 +-
.../config_source/LocalDirConfig.py | 40 ++++++------
.../config_source/RemoteGitConfig.py | 51 +++++++++------
.../config_operator/k8s/K8sConfigMap.py | 26 +++++---
config_operator/config_operator/main.py | 12 +++-
config_operator/containers/k8s/git-repo-test.yml | 9 +++
config_operator/requirements.txt | 1 +
.../tests/config_source/test_LocalDirConfig.py | 72 ++++++++++++++++++++++
.../tests/config_source/test_RemoteGitConfig.py | 49 +++++++++++++++
config_operator/tests/k8s/test_K8sConfigMap.py | 53 +++++++++++++++-
.../resources/localDirBadTest/collections.yml | 2 +
.../tests/resources/localDirTest/.hidden_file.txt | 1 +
.../tests/resources/localDirTest/README.md | 1 +
.../tests/resources/localDirTest/collections.yml | 1 +
14 files changed, 269 insertions(+), 55 deletions(-)
diff --git a/config_operator/README.md b/config_operator/README.md
index 4624a0f..ba4c6fc 100644
--- a/config_operator/README.md
+++ b/config_operator/README.md
@@ -16,16 +16,16 @@ The component runs as a kubernetes operator (see containerization section)
pip install -e .
pytest -d
-# Containerizaion
+# Containerization
## Docker
- docker build . -f containers/docker/Dockerfile --no-cache --tag tloubrieu/config-operator:latest
+ docker build . -f containers/docker/Dockerfile -t nexusjpl/config-operator:latest
To publish the docker image on dockerhub do (step necessary for kubernetes deployment):
docker login
- docker push tloubrieu/sdap-ingest-manager:latest
+ docker push nexusjpl/config-operator:latest
## Kubernetes
diff --git a/config_operator/config_operator/config_source/LocalDirConfig.py b/config_operator/config_operator/config_source/LocalDirConfig.py
index cf95f42..555a7fa 100644
--- a/config_operator/config_operator/config_source/LocalDirConfig.py
+++ b/config_operator/config_operator/config_source/LocalDirConfig.py
@@ -1,10 +1,11 @@
+import asyncio
import os
import time
import logging
+from functools import partial
import yaml
from typing import Callable
-
from config_operator.config_source.exceptions import UnreadableFileException
logging.basicConfig(level=logging.DEBUG)
@@ -16,12 +17,14 @@ LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1
class LocalDirConfig:
def __init__(self, local_dir: str,
- update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS):
+ update_every_seconds: float = LISTEN_FOR_UPDATE_INTERVAL_SECONDS,
+ update_date_fun=os.path.getmtime):
logger.info(f'create config on local dir {local_dir}')
self._local_dir = local_dir
+ self._update_date_fun = update_date_fun
+ self._update_every_seconds = update_every_seconds
self._latest_update = self._get_latest_update()
- self._update_every_seconds=update_every_seconds
-
+
def get_files(self):
files = []
for f in os.listdir(self._local_dir):
@@ -49,28 +52,29 @@ class LocalDirConfig:
raise UnreadableFileException(e)
except yaml.parser.ParserError as e:
raise UnreadableFileException(e)
-
+ except yaml.scanner.ScannerError as e:
+ raise UnreadableFileException(e)
def _get_latest_update(self):
- m_times = [os.path.getmtime(root) for root, _, _ in os.walk(self._local_dir)]
+ m_times = [self._update_date_fun(root) for root, _, _ in os.walk(self._local_dir)]
if m_times:
- return time.ctime(max(m_times))
+ return max(m_times)
else:
return None
- def when_updated(self, callback: Callable[[], None]):
+ async def when_updated(self, callback: Callable[[], None], loop=None):
"""
call function callback when the local directory is updated.
"""
- while True:
- time.sleep(self._update_every_seconds)
- latest_update = self._get_latest_update()
- if latest_update is None or (latest_update > self._latest_update):
- logger.info("local config dir has been updated")
- callback()
- self._latest_update = latest_update
- else:
- logger.debug("local config dir has not been updated")
+ if loop is None:
+ loop = asyncio.get_running_loop()
- return None
+ latest_update = self._get_latest_update()
+ if latest_update is None or (latest_update > self._latest_update):
+ logger.info("local config dir has been updated")
+ callback()
+ else:
+ logger.debug("local config dir has not been updated")
+ loop.call_later(self._update_every_seconds, partial(self.when_updated, callback, loop))
+ return None
diff --git a/config_operator/config_operator/config_source/RemoteGitConfig.py b/config_operator/config_operator/config_source/RemoteGitConfig.py
index 17d8223..38cbe2d 100644
--- a/config_operator/config_operator/config_source/RemoteGitConfig.py
+++ b/config_operator/config_operator/config_source/RemoteGitConfig.py
@@ -1,9 +1,12 @@
+import asyncio
import logging
import os
import sys
-import time
-from git import Repo
+from functools import partial
from typing import Callable
+
+from git import Repo
+
from .LocalDirConfig import LocalDirConfig
logging.basicConfig(level=logging.DEBUG)
@@ -16,10 +19,11 @@ DEFAULT_LOCAL_REPO_DIR = os.path.join(sys.prefix, 'sdap', 'conf')
class RemoteGitConfig(LocalDirConfig):
def __init__(self, git_url: str,
git_branch: str = 'master',
+ git_username: str = None,
git_token: str = None,
- update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS,
- local_dir: str = DEFAULT_LOCAL_REPO_DIR
- ):
+ update_every_seconds: float = LISTEN_FOR_UPDATE_INTERVAL_SECONDS,
+ local_dir: str = DEFAULT_LOCAL_REPO_DIR,
+ repo: Repo = None):
"""
:param git_url:
@@ -27,14 +31,23 @@ class RemoteGitConfig(LocalDirConfig):
:param git_token:
"""
self._git_url = git_url if git_url.endswith(".git") else git_url + '.git'
+ if git_username and git_token:
+ self._git_url.replace('https://', f'https://{git_username}:{git_token}')
+ self._git_url.replace('http://', f'http://{git_username}:{git_token}')
+
self._git_branch = git_branch
self._git_token = git_token
if local_dir is None:
local_dir = DEFAULT_LOCAL_REPO_DIR
self._update_every_seconds = update_every_seconds
super().__init__(local_dir, update_every_seconds=self._update_every_seconds)
- self._repo = None
- self._init_local_config_repo()
+
+ if repo:
+ self._repo = repo
+ else:
+ self._repo = None
+ self._init_local_config_repo()
+
self._latest_commit_key = self._pull_remote()
def _pull_remote(self):
@@ -49,19 +62,21 @@ class RemoteGitConfig(LocalDirConfig):
self._repo.git.fetch()
self._repo.git.checkout(self._git_branch)
- def when_updated(self, callback: Callable[[], None]):
+ async def when_updated(self, callback: Callable[[], None], loop=None):
"""
call function callback when the remote git repository is updated.
"""
- while True:
- time.sleep(self._update_every_seconds)
- remote_commit_key = self._pull_remote()
- if remote_commit_key != self._latest_commit_key:
- logger.info("remote git repository has been updated")
- callback()
- self._latest_commit_key = remote_commit_key
- else:
- logger.debug("remote git repository has not been updated")
+ if loop is None:
+ loop = asyncio.get_running_loop()
- return None
+ remote_commit_key = self._pull_remote()
+ if remote_commit_key != self._latest_commit_key:
+ logger.info("remote git repository has been updated")
+ callback()
+ self._latest_commit_key = remote_commit_key
+ else:
+ logger.debug("remote git repository has not been updated")
+ loop.call_later(self._update_every_seconds, partial(self.when_updated, callback, loop))
+
+ return None
diff --git a/config_operator/config_operator/k8s/K8sConfigMap.py b/config_operator/config_operator/k8s/K8sConfigMap.py
index e2a7a10..40d07c9 100644
--- a/config_operator/config_operator/k8s/K8sConfigMap.py
+++ b/config_operator/config_operator/k8s/K8sConfigMap.py
@@ -4,7 +4,8 @@ from kubernetes import client, config
from config_operator.config_source import LocalDirConfig, RemoteGitConfig
from kubernetes.client.rest import ApiException
from typing import Union
-
+from kubernetes.client.api.core_v1_api import CoreV1Api
+from kubernetes.client import ApiClient
from config_operator.config_source.exceptions import UnreadableFileException
logging.basicConfig(level=logging.INFO)
@@ -14,19 +15,24 @@ logger = logging.getLogger(__name__)
class K8sConfigMap:
def __init__(self, configmap_name: str,
namespace: str,
- external_config: Union[LocalDirConfig, RemoteGitConfig]):
+ external_config: Union[LocalDirConfig, RemoteGitConfig],
+ api_instance: ApiClient = None,
+ api_core_v1_instance: CoreV1Api = None):
self._git_remote_config = external_config
self._namespace = namespace
self._configmap_name = configmap_name
- # test is this runs inside kubernetes cluster
- if os.getenv('KUBERNETES_SERVICE_HOST'):
- config.load_incluster_config()
- else:
- config.load_kube_config()
- configuration = client.Configuration()
- self._api_instance = client.ApiClient(configuration)
- self._api_core_v1_instance = client.CoreV1Api(self._api_instance)
+ if api_core_v1_instance is None:
+ # test is this runs inside kubernetes cluster
+ if os.getenv('KUBERNETES_SERVICE_HOST'):
+ config.load_incluster_config()
+ else:
+ config.load_kube_config()
+ configuration = client.Configuration()
+ api_instance = client.ApiClient(configuration)
+ api_core_v1_instance = client.CoreV1Api(api_instance)
+ self._api_instance = api_instance
+ self._api_core_v1_instance = api_core_v1_instance
self.publish()
def __del__(self):
diff --git a/config_operator/config_operator/main.py b/config_operator/config_operator/main.py
index db4dbcb..fac6741 100644
--- a/config_operator/config_operator/main.py
+++ b/config_operator/config_operator/main.py
@@ -1,4 +1,5 @@
import logging
+import asyncio
import kopf
from config_operator.config_source import RemoteGitConfig
from config_operator.k8s import K8sConfigMap
@@ -6,10 +7,10 @@ from config_operator.k8s import K8sConfigMap
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
+
@kopf.on.create('sdap.apache.org', 'v1', 'gitbasedconfigs')
def create_fn(body, spec, **kwargs):
# Get info from Git Repo Config object
- name = body['metadata']['name']
namespace = body['metadata']['namespace']
if 'git-url' not in spec.keys():
@@ -23,7 +24,7 @@ def create_fn(body, spec, **kwargs):
logger.info(f'config-map = {config_map}')
_kargs = {}
- for k in {'git-branch', 'git-token', 'update-every-seconds'}:
+ for k in {'git-branch', 'git-username', 'git-token', 'update-every-seconds'}:
if k in spec:
logger.info(f'{k} = {spec[k]}')
_kargs[k.replace('-', '_')] = spec[k]
@@ -32,7 +33,12 @@ def create_fn(body, spec, **kwargs):
config_map = K8sConfigMap(config_map, namespace, config)
- config.when_updated(config_map.publish)
+ asyncio.run(config.when_updated(config_map.publish))
msg = f"configmap {config_map} created from git repo {git_url}"
return {'message': msg}
+
+
+@kopf.on.login()
+def login_fn(**kwargs):
+ return kopf.login_via_client(**kwargs)
diff --git a/config_operator/containers/k8s/git-repo-test.yml b/config_operator/containers/k8s/git-repo-test.yml
new file mode 100644
index 0000000..6a98454
--- /dev/null
+++ b/config_operator/containers/k8s/git-repo-test.yml
@@ -0,0 +1,9 @@
+apiVersion: sdap.apache.org/v1
+kind: gitBasedConfig
+metadata:
+ name: collections-config-gitcfg
+spec:
+ git-url: https://github.com/tloubrieu-jpl/sdap-ingester-config.git
+ git-branch: master
+ git-token: whatever
+ config-map: my-configmap
\ No newline at end of file
diff --git a/config_operator/requirements.txt b/config_operator/requirements.txt
index 5d452e2..84ac622 100644
--- a/config_operator/requirements.txt
+++ b/config_operator/requirements.txt
@@ -2,3 +2,4 @@ GitPython==3.1.2
kubernetes==11.0
kopf==0.26
+
diff --git a/config_operator/tests/config_source/test_LocalDirConfig.py b/config_operator/tests/config_source/test_LocalDirConfig.py
new file mode 100644
index 0000000..fed13c0
--- /dev/null
+++ b/config_operator/tests/config_source/test_LocalDirConfig.py
@@ -0,0 +1,72 @@
+import asyncio
+import os
+import time
+import unittest
+from datetime import datetime
+from unittest.mock import Mock
+
+from config_operator.config_source import LocalDirConfig
+from config_operator.config_source.exceptions import UnreadableFileException
+
+
+class TestLocalDirConfig(unittest.TestCase):
+ def test_get_files(self):
+ local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest")
+ local_dir_config = LocalDirConfig(local_dir)
+ files = local_dir_config.get_files()
+ self.assertEqual(len(files), 1)
+ self.assertEqual(files[0], 'collections.yml')
+
+ def test_get_good_file_content(self):
+ local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest")
+ local_dir_config = LocalDirConfig(local_dir)
+ files = local_dir_config.get_files()
+ content = local_dir_config.get_file_content(files[0])
+ self.assertEqual(content.strip(), 'test: 1')
+
+ def test_get_bad_file_content(self):
+ unreadable_file = False
+ try:
+ local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirBadTest")
+ local_dir_config = LocalDirConfig(local_dir)
+ files = local_dir_config.get_files()
+ content = local_dir_config.get_file_content(files[0])
+ except UnreadableFileException as e:
+ unreadable_file = True
+
+ finally:
+ self.assertTrue(unreadable_file)
+
+ def test_when_updated(self):
+
+ callback = Mock()
+ local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest")
+
+ local_dir_config = LocalDirConfig(local_dir,
+ update_every_seconds=0.25,
+ update_date_fun=lambda x: datetime.now().timestamp())
+
+ asyncio.run(local_dir_config.when_updated(callback))
+
+ time.sleep(2)
+
+ assert callback.called
+
+ def test_when_not_updated(self):
+
+ callback = Mock()
+ local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest")
+
+ local_dir_config = LocalDirConfig(local_dir,
+ update_every_seconds=0.25,
+ update_date_fun=lambda x: 0)
+
+ asyncio.run(local_dir_config.when_updated(callback))
+
+ time.sleep(2)
+
+ assert not callback.called
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/config_operator/tests/config_source/test_RemoteGitConfig.py b/config_operator/tests/config_source/test_RemoteGitConfig.py
new file mode 100644
index 0000000..52e6c9d
--- /dev/null
+++ b/config_operator/tests/config_source/test_RemoteGitConfig.py
@@ -0,0 +1,49 @@
+import asyncio
+import os
+import tempfile
+import time
+import unittest
+from unittest.mock import Mock
+
+from config_operator.config_source import RemoteGitConfig
+
+
+class TestRemoteDirConfig(unittest.TestCase):
+ _latest_commit = 0
+
+ def _get_origin(self):
+ commit = Mock()
+ commit.hexsha = self._latest_commit
+ self._latest_commit += 1
+
+ pull_result = Mock()
+ pull_result.commit = commit
+
+ return [pull_result]
+
+ def test_when_updated(self):
+ origin_branch = Mock()
+ origin_branch.pull = self._get_origin
+
+ remotes = Mock()
+ remotes.origin = origin_branch
+
+ repo = Mock()
+ repo.remotes = remotes
+
+ git_config = RemoteGitConfig(git_url='https://github.com/tloubrieu-jpl/sdap-ingester-config',
+ update_every_seconds=0.25,
+ local_dir=os.path.join(tempfile.gettempdir(), 'sdap'),
+ repo=repo)
+
+ callback = Mock()
+
+ asyncio.run(git_config.when_updated(callback))
+
+ time.sleep(2)
+
+ assert callback.called
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/config_operator/tests/k8s/test_K8sConfigMap.py b/config_operator/tests/k8s/test_K8sConfigMap.py
index 1660e69..e649fbc 100644
--- a/config_operator/tests/k8s/test_K8sConfigMap.py
+++ b/config_operator/tests/k8s/test_K8sConfigMap.py
@@ -1,11 +1,19 @@
import unittest
+from unittest.mock import Mock
import os
+from kubernetes.client.rest import ApiException
from config_operator.k8s import K8sConfigMap
from config_operator.config_source import RemoteGitConfig, LocalDirConfig
+if 'GIT_USERNAME' in os.environ:
+ GIT_USERNAME = os.environ['GIT_USERNAME']
+if 'GIT_TOKEN' in os.environ:
+ GIT_TOKEN = os.environ['GIT_TOKEN']
+
class K8sConfigMapTest(unittest.TestCase):
+ @unittest.skip('requires remote git')
def test_createconfigmapfromgit(self):
remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config")
@@ -13,13 +21,52 @@ class K8sConfigMapTest(unittest.TestCase):
config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config)
config_map.publish()
+ @unittest.skip('requires remote git')
+ def test_createconfigmapfromgit_with_token(self):
+ remote_git_config = RemoteGitConfig("https://podaac-git.jpl.nasa.gov:8443/podaac-sdap/deployment-configs.git",
+ git_username=GIT_USERNAME,
+ git_token=GIT_TOKEN)
+
+ config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config)
+ config_map.publish()
+
+ @unittest.skip('requires kubernetes')
+ def test_createconfigmapfromlocaldir_with_k8s(self):
+ local_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ '..',
+ 'resources',
+ 'localDirTest')
+ local_config = LocalDirConfig(local_dir)
+
+ config_map = K8sConfigMap('collection-ingester', 'sdap', local_config)
+ config_map.publish()
+
def test_createconfigmapfromlocaldir(self):
local_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'..',
- 'resources')
- remote_git_config = LocalDirConfig(local_dir)
+ 'resources',
+ 'localDirTest')
+ local_config = LocalDirConfig(local_dir)
- config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config)
+ api_instance = Mock()
+ api_instance.close = Mock()
+
+ api_core_v1_mock = Mock()
+ api_core_v1_mock.create_namespaced_config_map = Mock(return_value={
+ 'api_version': 'v1',
+ 'binary_data': None,
+ 'data': {}
+ })
+ api_core_v1_mock.patch_namespaced_config_map = Mock(return_value={
+ 'api_version': 'v1',
+ 'binary_data': None,
+ 'data': {}
+ })
+ api_core_v1_mock.create_namespaced_config_map.side_effect = Mock(side_effect=ApiException('409'))
+
+ config_map = K8sConfigMap('collection-ingester', 'sdap', local_config,
+ api_instance = api_instance,
+ api_core_v1_instance=api_core_v1_mock)
config_map.publish()
diff --git a/config_operator/tests/resources/localDirBadTest/collections.yml b/config_operator/tests/resources/localDirBadTest/collections.yml
new file mode 100644
index 0000000..7903016
--- /dev/null
+++ b/config_operator/tests/resources/localDirBadTest/collections.yml
@@ -0,0 +1,2 @@
+test:
+test
diff --git a/config_operator/tests/resources/localDirTest/.hidden_file.txt b/config_operator/tests/resources/localDirTest/.hidden_file.txt
new file mode 100644
index 0000000..30d74d2
--- /dev/null
+++ b/config_operator/tests/resources/localDirTest/.hidden_file.txt
@@ -0,0 +1 @@
+test
\ No newline at end of file
diff --git a/config_operator/tests/resources/localDirTest/README.md b/config_operator/tests/resources/localDirTest/README.md
new file mode 100644
index 0000000..30d74d2
--- /dev/null
+++ b/config_operator/tests/resources/localDirTest/README.md
@@ -0,0 +1 @@
+test
\ No newline at end of file
diff --git a/config_operator/tests/resources/localDirTest/collections.yml b/config_operator/tests/resources/localDirTest/collections.yml
new file mode 100644
index 0000000..4857bf6
--- /dev/null
+++ b/config_operator/tests/resources/localDirTest/collections.yml
@@ -0,0 +1 @@
+test: 1