You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2020/07/21 16:28:35 UTC
[incubator-sdap-ingester] branch kopf-operator updated (bbf81c8 ->
b0d49ab)
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a change to branch kopf-operator
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git.
discard bbf81c8 add unit tests and support for git username/token
omit 5af240b SDAP-259: The Collection Manager now reloads the Collections Config file on an interval instead of watching for filesystem events
add 09f53d5 SDAP-259: The Collection Manager now reloads the Collections Config file on an interval instead of watching for filesystem events (#5)
new b0d49ab add unit tests and support for git username/token
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (bbf81c8)
\
N -- N -- N refs/heads/kopf-operator (b0d49ab)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
[incubator-sdap-ingester] 01/01: add unit tests and support for git
username/token
Posted by tl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch kopf-operator
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit b0d49ab712b9a100e4ce335ef25d8da853db3593
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Mon Jul 20 20:02:43 2020 -0700
add unit tests and support for git username/token
add login for kopf operator (helps for integration in k8s different versions)
add unit tests for config_source
support username/token
add unit test for K8sConfigMap
---
config_operator/README.md | 2 +-
.../config_source/LocalDirConfig.py | 37 +++++++-----
.../config_source/RemoteGitConfig.py | 44 +++++++++-----
.../config_operator/k8s/K8sConfigMap.py | 26 ++++----
config_operator/config_operator/main.py | 12 +++-
config_operator/containers/k8s/git-repo-test.yml | 9 +++
config_operator/requirements.txt | 1 +
.../tests/config_source/test_LocalDirConfig.py | 70 ++++++++++++++++++++++
.../tests/config_source/test_RemoteGitConfig.py | 52 ++++++++++++++++
config_operator/tests/k8s/test_K8sConfigMap.py | 53 +++++++++++++++-
.../resources/localDirBadTest/collections.yml | 2 +
.../tests/resources/localDirTest/.hidden_file.txt | 1 +
.../tests/resources/localDirTest/README.md | 1 +
.../tests/resources/localDirTest/collections.yml | 1 +
14 files changed, 264 insertions(+), 47 deletions(-)
diff --git a/config_operator/README.md b/config_operator/README.md
index 4624a0f..91ca05f 100644
--- a/config_operator/README.md
+++ b/config_operator/README.md
@@ -25,7 +25,7 @@ The component runs as a kubernetes operator (see containerization section)
To publish the docker image on dockerhub do (step necessary for kubernetes deployment):
docker login
- docker push tloubrieu/sdap-ingest-manager:latest
+ docker push tloubrieu/config-operator:latest
## Kubernetes
diff --git a/config_operator/config_operator/config_source/LocalDirConfig.py b/config_operator/config_operator/config_source/LocalDirConfig.py
index cf95f42..e31a0d7 100644
--- a/config_operator/config_operator/config_source/LocalDirConfig.py
+++ b/config_operator/config_operator/config_source/LocalDirConfig.py
@@ -1,6 +1,8 @@
+import asyncio
import os
import time
import logging
+from functools import partial
import yaml
from typing import Callable
@@ -16,11 +18,14 @@ LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1
class LocalDirConfig:
def __init__(self, local_dir: str,
- update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS):
+ update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS,
+ update_date_fun=os.path.getmtime):
logger.info(f'create config on local dir {local_dir}')
self._local_dir = local_dir
- self._latest_update = self._get_latest_update()
+ self._update_date_fun = update_date_fun
self._update_every_seconds=update_every_seconds
+ self._latest_update = self._get_latest_update()
+
def get_files(self):
files = []
@@ -49,28 +54,30 @@ class LocalDirConfig:
raise UnreadableFileException(e)
except yaml.parser.ParserError as e:
raise UnreadableFileException(e)
-
+ except yaml.scanner.ScannerError as e:
+ raise UnreadableFileException(e)
def _get_latest_update(self):
- m_times = [os.path.getmtime(root) for root, _, _ in os.walk(self._local_dir)]
+ m_times = [self._update_date_fun(root) for root, _, _ in os.walk(self._local_dir)]
if m_times:
- return time.ctime(max(m_times))
+ return max(m_times)
else:
return None
- def when_updated(self, callback: Callable[[], None]):
+ async def when_updated(self, callback: Callable[[], None], loop=None):
"""
call function callback when the local directory is updated.
"""
- while True:
- time.sleep(self._update_every_seconds)
- latest_update = self._get_latest_update()
- if latest_update is None or (latest_update > self._latest_update):
- logger.info("local config dir has been updated")
- callback()
- self._latest_update = latest_update
- else:
- logger.debug("local config dir has not been updated")
+ if loop is None:
+ loop = asyncio.get_running_loop()
+
+ latest_update = self._get_latest_update()
+ if latest_update is None or (latest_update > self._latest_update):
+ logger.info("local config dir has been updated")
+ callback()
+ else:
+ logger.debug("local config dir has not been updated")
+ loop.call_later(self._update_every_seconds, partial(self.when_updated, callback, loop))
return None
diff --git a/config_operator/config_operator/config_source/RemoteGitConfig.py b/config_operator/config_operator/config_source/RemoteGitConfig.py
index 17d8223..2385b17 100644
--- a/config_operator/config_operator/config_source/RemoteGitConfig.py
+++ b/config_operator/config_operator/config_source/RemoteGitConfig.py
@@ -1,7 +1,8 @@
import logging
import os
import sys
-import time
+import asyncio
+from functools import partial
from git import Repo
from typing import Callable
from .LocalDirConfig import LocalDirConfig
@@ -16,10 +17,11 @@ DEFAULT_LOCAL_REPO_DIR = os.path.join(sys.prefix, 'sdap', 'conf')
class RemoteGitConfig(LocalDirConfig):
def __init__(self, git_url: str,
git_branch: str = 'master',
+ git_username: str = None,
git_token: str = None,
update_every_seconds: int = LISTEN_FOR_UPDATE_INTERVAL_SECONDS,
- local_dir: str = DEFAULT_LOCAL_REPO_DIR
- ):
+ local_dir: str = DEFAULT_LOCAL_REPO_DIR,
+ repo: Repo = None):
"""
:param git_url:
@@ -27,14 +29,23 @@ class RemoteGitConfig(LocalDirConfig):
:param git_token:
"""
self._git_url = git_url if git_url.endswith(".git") else git_url + '.git'
+ if git_username and git_token:
+ self._git_url.replace('https://', f'https://{git_username}:{git_token}')
+ self._git_url.replace('http://', f'http://{git_username}:{git_token}')
+
self._git_branch = git_branch
self._git_token = git_token
if local_dir is None:
local_dir = DEFAULT_LOCAL_REPO_DIR
self._update_every_seconds = update_every_seconds
super().__init__(local_dir, update_every_seconds=self._update_every_seconds)
- self._repo = None
- self._init_local_config_repo()
+
+ if repo:
+ self._repo = repo
+ else:
+ self._repo = None
+ self._init_local_config_repo()
+
self._latest_commit_key = self._pull_remote()
def _pull_remote(self):
@@ -49,19 +60,22 @@ class RemoteGitConfig(LocalDirConfig):
self._repo.git.fetch()
self._repo.git.checkout(self._git_branch)
- def when_updated(self, callback: Callable[[], None]):
+ async def when_updated(self, callback: Callable[[], None], loop=None):
"""
call function callback when the remote git repository is updated.
"""
- while True:
- time.sleep(self._update_every_seconds)
- remote_commit_key = self._pull_remote()
- if remote_commit_key != self._latest_commit_key:
- logger.info("remote git repository has been updated")
- callback()
- self._latest_commit_key = remote_commit_key
- else:
- logger.debug("remote git repository has not been updated")
+ if loop is None:
+ loop = asyncio.get_running_loop()
+
+ remote_commit_key = self._pull_remote()
+ if remote_commit_key != self._latest_commit_key:
+ logger.info("remote git repository has been updated")
+ callback()
+ self._latest_commit_key = remote_commit_key
+ else:
+ logger.debug("remote git repository has not been updated")
+
+ loop.call_later(self._update_every_seconds, partial(self.when_updated, callback, loop))
return None
diff --git a/config_operator/config_operator/k8s/K8sConfigMap.py b/config_operator/config_operator/k8s/K8sConfigMap.py
index e2a7a10..40d07c9 100644
--- a/config_operator/config_operator/k8s/K8sConfigMap.py
+++ b/config_operator/config_operator/k8s/K8sConfigMap.py
@@ -4,7 +4,8 @@ from kubernetes import client, config
from config_operator.config_source import LocalDirConfig, RemoteGitConfig
from kubernetes.client.rest import ApiException
from typing import Union
-
+from kubernetes.client.api.core_v1_api import CoreV1Api
+from kubernetes.client import ApiClient
from config_operator.config_source.exceptions import UnreadableFileException
logging.basicConfig(level=logging.INFO)
@@ -14,19 +15,24 @@ logger = logging.getLogger(__name__)
class K8sConfigMap:
def __init__(self, configmap_name: str,
namespace: str,
- external_config: Union[LocalDirConfig, RemoteGitConfig]):
+ external_config: Union[LocalDirConfig, RemoteGitConfig],
+ api_instance: ApiClient = None,
+ api_core_v1_instance: CoreV1Api = None):
self._git_remote_config = external_config
self._namespace = namespace
self._configmap_name = configmap_name
- # test is this runs inside kubernetes cluster
- if os.getenv('KUBERNETES_SERVICE_HOST'):
- config.load_incluster_config()
- else:
- config.load_kube_config()
- configuration = client.Configuration()
- self._api_instance = client.ApiClient(configuration)
- self._api_core_v1_instance = client.CoreV1Api(self._api_instance)
+ if api_core_v1_instance is None:
+ # test is this runs inside kubernetes cluster
+ if os.getenv('KUBERNETES_SERVICE_HOST'):
+ config.load_incluster_config()
+ else:
+ config.load_kube_config()
+ configuration = client.Configuration()
+ api_instance = client.ApiClient(configuration)
+ api_core_v1_instance = client.CoreV1Api(api_instance)
+ self._api_instance = api_instance
+ self._api_core_v1_instance = api_core_v1_instance
self.publish()
def __del__(self):
diff --git a/config_operator/config_operator/main.py b/config_operator/config_operator/main.py
index db4dbcb..fac6741 100644
--- a/config_operator/config_operator/main.py
+++ b/config_operator/config_operator/main.py
@@ -1,4 +1,5 @@
import logging
+import asyncio
import kopf
from config_operator.config_source import RemoteGitConfig
from config_operator.k8s import K8sConfigMap
@@ -6,10 +7,10 @@ from config_operator.k8s import K8sConfigMap
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
+
@kopf.on.create('sdap.apache.org', 'v1', 'gitbasedconfigs')
def create_fn(body, spec, **kwargs):
# Get info from Git Repo Config object
- name = body['metadata']['name']
namespace = body['metadata']['namespace']
if 'git-url' not in spec.keys():
@@ -23,7 +24,7 @@ def create_fn(body, spec, **kwargs):
logger.info(f'config-map = {config_map}')
_kargs = {}
- for k in {'git-branch', 'git-token', 'update-every-seconds'}:
+ for k in {'git-branch', 'git-username', 'git-token', 'update-every-seconds'}:
if k in spec:
logger.info(f'{k} = {spec[k]}')
_kargs[k.replace('-', '_')] = spec[k]
@@ -32,7 +33,12 @@ def create_fn(body, spec, **kwargs):
config_map = K8sConfigMap(config_map, namespace, config)
- config.when_updated(config_map.publish)
+ asyncio.run(config.when_updated(config_map.publish))
msg = f"configmap {config_map} created from git repo {git_url}"
return {'message': msg}
+
+
+@kopf.on.login()
+def login_fn(**kwargs):
+ return kopf.login_via_client(**kwargs)
diff --git a/config_operator/containers/k8s/git-repo-test.yml b/config_operator/containers/k8s/git-repo-test.yml
new file mode 100644
index 0000000..6a98454
--- /dev/null
+++ b/config_operator/containers/k8s/git-repo-test.yml
@@ -0,0 +1,9 @@
+apiVersion: sdap.apache.org/v1
+kind: gitBasedConfig
+metadata:
+ name: collections-config-gitcfg
+spec:
+ git-url: https://github.com/tloubrieu-jpl/sdap-ingester-config.git
+ git-branch: master
+ git-token: whatever
+ config-map: my-configmap
\ No newline at end of file
diff --git a/config_operator/requirements.txt b/config_operator/requirements.txt
index 5d452e2..84ac622 100644
--- a/config_operator/requirements.txt
+++ b/config_operator/requirements.txt
@@ -2,3 +2,4 @@ GitPython==3.1.2
kubernetes==11.0
kopf==0.26
+
diff --git a/config_operator/tests/config_source/test_LocalDirConfig.py b/config_operator/tests/config_source/test_LocalDirConfig.py
new file mode 100644
index 0000000..1c48356
--- /dev/null
+++ b/config_operator/tests/config_source/test_LocalDirConfig.py
@@ -0,0 +1,70 @@
+import asyncio
+import unittest
+from unittest.mock import Mock
+from unittest.mock import patch
+import os
+import time
+from datetime import datetime
+from config_operator.config_source import LocalDirConfig
+from config_operator.config_source.exceptions import UnreadableFileException
+
+
+
+class MyTestCase(unittest.TestCase):
+ def test_get_files(self):
+ local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest")
+ local_dir_config = LocalDirConfig(local_dir)
+ files = local_dir_config.get_files()
+ self.assertEqual(len(files), 1)
+ self.assertEqual(files[0], 'collections.yml')
+
+ def test_get_good_file_content(self):
+ local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest")
+ local_dir_config = LocalDirConfig(local_dir)
+ files = local_dir_config.get_files()
+ content = local_dir_config.get_file_content(files[0])
+ self.assertEqual(content.strip(), 'test: 1')
+
+ def test_get_bad_file_content(self):
+ unreadable_file = False
+ try:
+ local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirBadTest")
+ local_dir_config = LocalDirConfig(local_dir)
+ files = local_dir_config.get_files()
+ content = local_dir_config.get_file_content(files[0])
+ except UnreadableFileException as e:
+ unreadable_file = True
+
+ finally:
+ self.assertTrue(unreadable_file)
+
+ def test_when_udated(self):
+
+ callback = Mock()
+ local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest")
+
+ local_dir_config = LocalDirConfig(local_dir, update_every_seconds=0.25, update_date_fun=lambda x: datetime.now().timestamp())
+
+ asyncio.run(local_dir_config.when_updated(callback))
+
+ time.sleep(2)
+
+ assert callback.called
+
+ def test_when_not_udated(self):
+
+ callback = Mock()
+ local_dir = os.path.join(os.path.dirname(__file__), "../resources/localDirTest")
+
+ local_dir_config = LocalDirConfig(local_dir, update_every_seconds=0.25,
+ update_date_fun=lambda x: 0)
+
+ asyncio.run(local_dir_config.when_updated(callback))
+
+ time.sleep(2)
+
+ assert not callback.called
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/config_operator/tests/config_source/test_RemoteGitConfig.py b/config_operator/tests/config_source/test_RemoteGitConfig.py
new file mode 100644
index 0000000..50596d0
--- /dev/null
+++ b/config_operator/tests/config_source/test_RemoteGitConfig.py
@@ -0,0 +1,52 @@
+import unittest
+from unittest.mock import MagicMock, Mock, patch
+import asyncio
+import os
+import tempfile
+import time
+from git import Repo
+from config_operator.config_source import RemoteGitConfig
+
+class MyTestCase(unittest.TestCase):
+
+ _latest_commit = 0
+
+ def _get_origin(self):
+ commit = Mock()
+ commit.hexsha = self._latest_commit
+ self._latest_commit += 1
+
+ pull_result = Mock()
+ pull_result.commit = commit
+
+ return [pull_result]
+
+
+ def test_when_updated(self):
+
+ origin_branch = Mock()
+ origin_branch.pull = self._get_origin
+
+ remotes = Mock()
+ remotes.origin = origin_branch
+
+ repo = Mock()
+ repo.remotes = remotes
+
+ git_config = RemoteGitConfig(git_url='https://github.com/tloubrieu-jpl/sdap-ingester-config',
+ update_every_seconds=0.25,
+ local_dir=os.path.join(tempfile.gettempdir(), 'sdap'),
+ repo=repo)
+
+ callback = Mock()
+
+ asyncio.run(git_config.when_updated(callback))
+
+ time.sleep(2)
+
+ assert callback.called
+
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/config_operator/tests/k8s/test_K8sConfigMap.py b/config_operator/tests/k8s/test_K8sConfigMap.py
index 1660e69..e649fbc 100644
--- a/config_operator/tests/k8s/test_K8sConfigMap.py
+++ b/config_operator/tests/k8s/test_K8sConfigMap.py
@@ -1,11 +1,19 @@
import unittest
+from unittest.mock import Mock
import os
+from kubernetes.client.rest import ApiException
from config_operator.k8s import K8sConfigMap
from config_operator.config_source import RemoteGitConfig, LocalDirConfig
+if 'GIT_USERNAME' in os.environ:
+ GIT_USERNAME = os.environ['GIT_USERNAME']
+if 'GIT_TOKEN' in os.environ:
+ GIT_TOKEN = os.environ['GIT_TOKEN']
+
class K8sConfigMapTest(unittest.TestCase):
+ @unittest.skip('requires remote git')
def test_createconfigmapfromgit(self):
remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config")
@@ -13,13 +21,52 @@ class K8sConfigMapTest(unittest.TestCase):
config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config)
config_map.publish()
+ @unittest.skip('requires remote git')
+ def test_createconfigmapfromgit_with_token(self):
+ remote_git_config = RemoteGitConfig("https://podaac-git.jpl.nasa.gov:8443/podaac-sdap/deployment-configs.git",
+ git_username=GIT_USERNAME,
+ git_token=GIT_TOKEN)
+
+ config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config)
+ config_map.publish()
+
+ @unittest.skip('requires kubernetes')
+ def test_createconfigmapfromlocaldir_with_k8s(self):
+ local_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ '..',
+ 'resources',
+ 'localDirTest')
+ local_config = LocalDirConfig(local_dir)
+
+ config_map = K8sConfigMap('collection-ingester', 'sdap', local_config)
+ config_map.publish()
+
def test_createconfigmapfromlocaldir(self):
local_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'..',
- 'resources')
- remote_git_config = LocalDirConfig(local_dir)
+ 'resources',
+ 'localDirTest')
+ local_config = LocalDirConfig(local_dir)
- config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config)
+ api_instance = Mock()
+ api_instance.close = Mock()
+
+ api_core_v1_mock = Mock()
+ api_core_v1_mock.create_namespaced_config_map = Mock(return_value={
+ 'api_version': 'v1',
+ 'binary_data': None,
+ 'data': {}
+ })
+ api_core_v1_mock.patch_namespaced_config_map = Mock(return_value={
+ 'api_version': 'v1',
+ 'binary_data': None,
+ 'data': {}
+ })
+ api_core_v1_mock.create_namespaced_config_map.side_effect = Mock(side_effect=ApiException('409'))
+
+ config_map = K8sConfigMap('collection-ingester', 'sdap', local_config,
+ api_instance = api_instance,
+ api_core_v1_instance=api_core_v1_mock)
config_map.publish()
diff --git a/config_operator/tests/resources/localDirBadTest/collections.yml b/config_operator/tests/resources/localDirBadTest/collections.yml
new file mode 100644
index 0000000..7903016
--- /dev/null
+++ b/config_operator/tests/resources/localDirBadTest/collections.yml
@@ -0,0 +1,2 @@
+test:
+test
diff --git a/config_operator/tests/resources/localDirTest/.hidden_file.txt b/config_operator/tests/resources/localDirTest/.hidden_file.txt
new file mode 100644
index 0000000..30d74d2
--- /dev/null
+++ b/config_operator/tests/resources/localDirTest/.hidden_file.txt
@@ -0,0 +1 @@
+test
\ No newline at end of file
diff --git a/config_operator/tests/resources/localDirTest/README.md b/config_operator/tests/resources/localDirTest/README.md
new file mode 100644
index 0000000..30d74d2
--- /dev/null
+++ b/config_operator/tests/resources/localDirTest/README.md
@@ -0,0 +1 @@
+test
\ No newline at end of file
diff --git a/config_operator/tests/resources/localDirTest/collections.yml b/config_operator/tests/resources/localDirTest/collections.yml
new file mode 100644
index 0000000..4857bf6
--- /dev/null
+++ b/config_operator/tests/resources/localDirTest/collections.yml
@@ -0,0 +1 @@
+test: 1