You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/06/10 20:39:06 UTC
[incubator-sdap-ingester] 05/08: move files in specific
subdirectory, start k8s deployment, does not work
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch config_map
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 49042bb822c2c2e9ab3eb26dd2d07e4b2992f383
Author: thomas loubrieu <th...@jpl.nasa.gov>
AuthorDate: Thu Jun 4 20:19:42 2020 -0700
move files in specific subdirectory, start k8s deployment, does not work
---
README.md | 11 -------
config_operator/README.md | 38 ++++++++++++++++++++++
config_operator/config_operator/__init__.py | 1 +
.../config_operator}/config_operator.py | 5 +--
.../config_source}/LocalDirConfig.py | 5 ++-
.../config_source}/RemoteGitConfig.py | 27 +++++++--------
.../config_operator/config_source/__init__.py | 2 ++
.../config_operator/config_source}/exceptions.py | 2 +-
.../config_operator/k8s/K8sConfigMap.py | 4 +--
config_operator/config_operator/k8s/__init__.py | 1 +
config_operator/containers/docker/Dockerfile | 10 ++++++
config_operator/containers/k8s/deployment.yml | 21 ++++++++++++
config_operator/requirements.txt | 3 ++
setup.py => config_operator/setup.py | 16 ++++-----
.../config => config_operator/tests}/__init__.py | 0
.../tests}/config/__init__.py | 0
config_operator/tests/config/test_ConfigMap.py | 27 +++++++++++++++
config_operator/tests/resources/collections.yml | 9 +++++
containers/docker/config-operator/Dockerfile | 10 ------
sdap_ingest_manager/config/__init__.py | 3 --
setup.py | 2 +-
tests/config/test_ConfigMap.py | 22 -------------
22 files changed, 141 insertions(+), 78 deletions(-)
diff --git a/README.md b/README.md
index d653212..adb6ddd 100644
--- a/README.md
+++ b/README.md
@@ -128,17 +128,6 @@ Deploy a local rabbitmq service, for example with docker.
### Launch the service
-#### The config operator:
-
-This component helps to import a configuration directory on local file system or on a git repository as a configMap in kubernetes.
-This makes the configuration easily accessible to all the nodes of the cluster whereas the configuration stays in a single place.
-The configurations can be updated while the service is running (-u). The configuration updates will be published to kubernetes pods by patching the existing configurations.
-
- config-operator -h
- config-operator -l tests/resources/data -n sdap -cm collection-ingester-config
- config-operator --git-url=https://github.com/tloubrieu-jpl/sdap-ingester-config --namespace=sdap --config-map=collection-ingester-config
-
-#### The collection ingestion service
The service reads the collection configuration and submit granule ingestion messages to the message broker (rabbitmq).
For each collection, 2 ingestion priority levels are proposed: the nominal priority, the priority for forward processing (newer files), usually higher.
diff --git a/config_operator/README.md b/config_operator/README.md
new file mode 100644
index 0000000..61889c4
--- /dev/null
+++ b/config_operator/README.md
@@ -0,0 +1,38 @@
+# Config Operator
+
+## Purpose
+
+Component which synchonizes local configuration in a directory, on a file system, or configuration files managed in a git repository with kubernetes configMap.
+This helps to make a configuration managed by the operators in a single place (git, host file system) available in the kubernetes cluster.
+
+For SDAP, it is used to make the configuration of the collections to be ingested available to the ingester service pods.
+
+# Launch the service
+
+The configurations can be updated while the service is running (-u). The configuration updates will be published to kubernetes pods by patching the existing configurations.
+
+ config-operator -h
+ config-operator -l tests/resources/data -n sdap -cm collection-ingester-config
+ config-operator --git-url=https://github.com/tloubrieu-jpl/sdap-ingester-config --namespace=sdap --config-map=collection-ingester-config
+
+# Developers
+
+ git clone ...
+ cd config_operator
+ pip install -e .
+ pytest -d
+
+# Containerizaion
+
+## Docker
+
+ docker build . -f containers/docker/Dockerfile --no-cache --tag tloubrieu/config-operator:latest
+
+To publish the docker image on dockerhub do (step necessary for kubernetes deployment):
+
+ docker login
+ docker push tloubrieu/sdap-ingest-manager:latest
+
+## Kubernetes
+
+ kubectl apply -f containers/k8s/deployment.yml -n sdap
\ No newline at end of file
diff --git a/config_operator/config_operator/__init__.py b/config_operator/config_operator/__init__.py
new file mode 100644
index 0000000..7d24cef
--- /dev/null
+++ b/config_operator/config_operator/__init__.py
@@ -0,0 +1 @@
+__version__ = '0.1.0.dev0'
diff --git a/sdap_ingest_manager/config_operator.py b/config_operator/config_operator/config_operator.py
similarity index 89%
rename from sdap_ingest_manager/config_operator.py
rename to config_operator/config_operator/config_operator.py
index 97699b9..b032384 100644
--- a/sdap_ingest_manager/config_operator.py
+++ b/config_operator/config_operator/config_operator.py
@@ -1,5 +1,6 @@
import argparse
-from sdap_ingest_manager.config import RemoteGitConfig, LocalDirConfig, K8ConfigMap
+from config_operator.config_source import RemoteGitConfig, LocalDirConfig
+from config_operator.k8s import K8sConfigMap
def main():
@@ -25,7 +26,7 @@ def main():
else:
config = RemoteGitConfig(options.git_url, branch=options.git_branch, token=options.git_token)
- config_map = K8ConfigMap(options.config_map, options.namespace, config)
+ config_map = K8sConfigMap(options.config_map, options.namespace, config)
config_map.publish()
if options.updated_continuously:
diff --git a/sdap_ingest_manager/config/LocalDirConfig.py b/config_operator/config_operator/config_source/LocalDirConfig.py
similarity index 93%
rename from sdap_ingest_manager/config/LocalDirConfig.py
rename to config_operator/config_operator/config_source/LocalDirConfig.py
index 6d8a7cf..89c0e5a 100644
--- a/sdap_ingest_manager/config/LocalDirConfig.py
+++ b/config_operator/config_operator/config_source/LocalDirConfig.py
@@ -3,7 +3,8 @@ import time
import logging
import yaml
-from sdap_ingest_manager.config.exceptions import UnreadableFileException
+
+from config_operator.config_source.exceptions import UnreadableFileException
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
@@ -12,7 +13,9 @@ LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 1
class LocalDirConfig:
+
def __init__(self, local_dir):
+ logger.info(f'create config on local dir {local_dir}')
self._local_dir = local_dir
self._latest_update = self._get_latest_update()
diff --git a/sdap_ingest_manager/config/RemoteGitConfig.py b/config_operator/config_operator/config_source/RemoteGitConfig.py
similarity index 77%
rename from sdap_ingest_manager/config/RemoteGitConfig.py
rename to config_operator/config_operator/config_source/RemoteGitConfig.py
index a344246..24e614a 100644
--- a/sdap_ingest_manager/config/RemoteGitConfig.py
+++ b/config_operator/config_operator/config_source/RemoteGitConfig.py
@@ -2,19 +2,19 @@ import logging
import os
import sys
import time
-from git import Repo, Remote
-from sdap_ingest_manager.config import LocalDirConfig
-
+from git import Repo
+from .LocalDirConfig import LocalDirConfig
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
LISTEN_FOR_UPDATE_INTERVAL_SECONDS = 5
+
class RemoteGitConfig(LocalDirConfig):
def __init__(self, git_url,
- git_branch='master',
- git_token=None
+ branch='master',
+ token=None
):
"""
@@ -23,18 +23,18 @@ class RemoteGitConfig(LocalDirConfig):
:param git_token:
"""
self._git_url = git_url if git_url.endswith(".git") else git_url + '.git'
- self._git_branch = git_branch
- self._git_token = git_token
+ self._git_branch = branch
+ self._git_token = token
local_dir = os.path.join(sys.prefix, 'sdap', 'conf')
super().__init__(local_dir)
self._repo = None
self._init_local_config_repo()
- self._latest_commit_key = self._repo.head.commit.hexsha
+ self._latest_commit_key = self._pull_remote()
def _pull_remote(self):
o = self._repo.remotes.origin
res = o.pull()
- return res[0].commit.hexsha # return the latest commit key
+ return res[0].commit.hexsha # return the latest commit key
def _init_local_config_repo(self):
self._repo = Repo.init(self._local_dir)
@@ -43,6 +43,8 @@ class RemoteGitConfig(LocalDirConfig):
self._repo.git.fetch()
self._repo.git.checkout(self._git_branch)
+
+
def when_updated(self, callback):
while True:
@@ -54,9 +56,4 @@ class RemoteGitConfig(LocalDirConfig):
self._latest_commit_key = remote_commit_key
else:
logger.debug("remote git repository has not been updated")
-
-
-
-
-
-
+
diff --git a/config_operator/config_operator/config_source/__init__.py b/config_operator/config_operator/config_source/__init__.py
new file mode 100644
index 0000000..b23e9be
--- /dev/null
+++ b/config_operator/config_operator/config_source/__init__.py
@@ -0,0 +1,2 @@
+from .RemoteGitConfig import RemoteGitConfig
+from .LocalDirConfig import LocalDirConfig
diff --git a/sdap_ingest_manager/config/exceptions.py b/config_operator/config_operator/config_source/exceptions.py
similarity index 83%
rename from sdap_ingest_manager/config/exceptions.py
rename to config_operator/config_operator/config_source/exceptions.py
index c06b881..96e8502 100644
--- a/sdap_ingest_manager/config/exceptions.py
+++ b/config_operator/config_operator/config_source/exceptions.py
@@ -1,4 +1,4 @@
class UnreadableFileException(Exception):
- pass
\ No newline at end of file
+ pass
diff --git a/sdap_ingest_manager/config/K8ConfigMap.py b/config_operator/config_operator/k8s/K8sConfigMap.py
similarity index 96%
rename from sdap_ingest_manager/config/K8ConfigMap.py
rename to config_operator/config_operator/k8s/K8sConfigMap.py
index e9afe55..b16b58c 100644
--- a/sdap_ingest_manager/config/K8ConfigMap.py
+++ b/config_operator/config_operator/k8s/K8sConfigMap.py
@@ -2,13 +2,13 @@ import logging
from kubernetes import client, config
from kubernetes.client.rest import ApiException
-from sdap_ingest_manager.config.exceptions import UnreadableFileException
+from config_operator.config_source.exceptions import UnreadableFileException
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
-class K8ConfigMap:
+class K8sConfigMap:
def __init__(self, configmap_name, namespace, git_remote_config):
self._git_remote_config = git_remote_config
self._namespace = namespace
diff --git a/config_operator/config_operator/k8s/__init__.py b/config_operator/config_operator/k8s/__init__.py
new file mode 100644
index 0000000..2d5a84d
--- /dev/null
+++ b/config_operator/config_operator/k8s/__init__.py
@@ -0,0 +1 @@
+from .K8sConfigMap import K8sConfigMap
diff --git a/config_operator/containers/docker/Dockerfile b/config_operator/containers/docker/Dockerfile
new file mode 100644
index 0000000..4e82c98
--- /dev/null
+++ b/config_operator/containers/docker/Dockerfile
@@ -0,0 +1,10 @@
+FROM python:3
+
+COPY /config_operator /config_operator/config_operator
+COPY /setup.py /config_operator/setup.py
+COPY /requirements.txt /config_operator/requirements.txt
+COPY /README.md /config_operator/README.md
+
+RUN cd /config_operator && pip install .
+
+CMD bash
diff --git a/config_operator/containers/k8s/deployment.yml b/config_operator/containers/k8s/deployment.yml
new file mode 100644
index 0000000..1501a75
--- /dev/null
+++ b/config_operator/containers/k8s/deployment.yml
@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: config-operator
+ labels:
+ app: sdap-config-operator
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: sdap-config-operator
+ template:
+ metadata:
+ labels:
+ app: sdap-config-operator
+ spec:
+ containers:
+ - name: sdap-config-operator
+ image: tloubrieu/config-operator:latest
+ imagePullPolicy: IfNotPresent
+ command: ['config-operator', '--git-url', 'https://github.com/tloubrieu-jpl/sdap-ingester-config' , '--namespace', 'sdap', '--config-map', 'collection-ingester-conf', '-u']
diff --git a/config_operator/requirements.txt b/config_operator/requirements.txt
new file mode 100644
index 0000000..4365d3d
--- /dev/null
+++ b/config_operator/requirements.txt
@@ -0,0 +1,3 @@
+GitPython==3.1.2
+kubernetes==11.0
+
diff --git a/setup.py b/config_operator/setup.py
similarity index 65%
copy from setup.py
copy to config_operator/setup.py
index f85d6a7..2d4c533 100644
--- a/setup.py
+++ b/config_operator/setup.py
@@ -1,15 +1,13 @@
import setuptools
-import os
-import subprocess
-import sys
import re
-PACKAGE_NAME = "sdap_ingest_manager"
+PACKAGE_NAME = "config_operator"
-with open("./sdap_ingest_manager/__init__.py") as fi:
+with open(f'./{PACKAGE_NAME}/__init__.py') as fi:
result = re.search(r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fi.read())
version = result.group(1)
+
with open("README.md", "r") as fh:
long_description = fh.read()
@@ -21,7 +19,7 @@ setuptools.setup(
version=version,
author="Apache - SDAP",
author_email="dev@sdap.apache.org",
- description="a helper to ingest data in sdap",
+ description="a service to synchronize git or local directory configuration with k8s configMap",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/tloubrieu-jpl/incubator-sdap-nexus-ingestion-manager",
@@ -33,10 +31,8 @@ setuptools.setup(
],
python_requires='>=3.6',
include_package_data=True,
- data_files=[('.sdap_ingest_manager/resources/', ['sdap_ingest_manager/resources/dataset_config_template.yml'])],
install_requires=requirements,
entry_points={
- 'console_scripts': ['config-operator=sdap_ingest_manager.config_operator:main',
- 'collection-ingester=sdap_ingest_manager.service:main']
- },
+ 'console_scripts': ['config-operator=config_operator.config_operator:main']
+ }
)
diff --git a/tests/config/__init__.py b/config_operator/tests/__init__.py
similarity index 100%
copy from tests/config/__init__.py
copy to config_operator/tests/__init__.py
diff --git a/tests/config/__init__.py b/config_operator/tests/config/__init__.py
similarity index 100%
rename from tests/config/__init__.py
rename to config_operator/tests/config/__init__.py
diff --git a/config_operator/tests/config/test_ConfigMap.py b/config_operator/tests/config/test_ConfigMap.py
new file mode 100644
index 0000000..4d339e3
--- /dev/null
+++ b/config_operator/tests/config/test_ConfigMap.py
@@ -0,0 +1,27 @@
+import unittest
+import os
+
+from config_operator.k8s import K8sConfigMap
+from config_operator.config_source import RemoteGitConfig, LocalDirConfig
+
+
+class ConfigMapTest(unittest.TestCase):
+ def test_createconfigmapfromgit(self):
+
+ remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config")
+
+ config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config)
+ config_map.publish()
+
+ def test_createconfigmapfromlocaldir(self):
+ local_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ '..',
+ 'resources')
+ remote_git_config = LocalDirConfig(local_dir)
+
+ config_map = K8sConfigMap('collection-ingester', 'sdap', remote_git_config)
+ config_map.publish()
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/config_operator/tests/resources/collections.yml b/config_operator/tests/resources/collections.yml
new file mode 100644
index 0000000..42d2fbc
--- /dev/null
+++ b/config_operator/tests/resources/collections.yml
@@ -0,0 +1,9 @@
+avhrr-oi-analysed-sst:
+ path: resources/history_manager/data/avhrr_oi/*.nc
+ variable: analysed_sst
+ priority: 9
+
+avhrr-oi-analysed-sst2:
+ path: resources/history_manager/data/avhrr_oi/*.nc
+ variable: analysed_sst
+ priority: 1
diff --git a/containers/docker/config-operator/Dockerfile b/containers/docker/config-operator/Dockerfile
deleted file mode 100644
index 81918b8..0000000
--- a/containers/docker/config-operator/Dockerfile
+++ /dev/null
@@ -1,10 +0,0 @@
-FROM python:3
-
-COPY /sdap_ingest_manager /sdap_ingest_manager/sdap_ingest_manager
-COPY /setup.py /sdap_ingest_manager/setup.py
-COPY /requirements.txt /sdap_ingest_manager/requirements.txt
-COPY /README.md /sdap_ingest_manager/README.md
-
-RUN cd /sdap_ingest_manager && pip install .
-
-CMD bash
diff --git a/sdap_ingest_manager/config/__init__.py b/sdap_ingest_manager/config/__init__.py
index 918967c..27a89b8 100644
--- a/sdap_ingest_manager/config/__init__.py
+++ b/sdap_ingest_manager/config/__init__.py
@@ -1,4 +1 @@
from sdap_ingest_manager.config.LocalConfiguration import LocalConfiguration
-from sdap_ingest_manager.config.K8ConfigMap import K8ConfigMap
-from sdap_ingest_manager.config.LocalDirConfig import LocalDirConfig
-from sdap_ingest_manager.config.RemoteGitConfig import RemoteGitConfig
\ No newline at end of file
diff --git a/setup.py b/setup.py
index f85d6a7..eeed194 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@ setuptools.setup(
data_files=[('.sdap_ingest_manager/resources/', ['sdap_ingest_manager/resources/dataset_config_template.yml'])],
install_requires=requirements,
entry_points={
- 'console_scripts': ['config-operator=sdap_ingest_manager.config_operator:main',
+ 'console_scripts': ['config-operator=config_operator.config_operator:main',
'collection-ingester=sdap_ingest_manager.service:main']
},
)
diff --git a/tests/config/test_ConfigMap.py b/tests/config/test_ConfigMap.py
deleted file mode 100644
index 2518b9a..0000000
--- a/tests/config/test_ConfigMap.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import unittest
-import os
-
-from flask import Flask
-from flask_restplus import Api
-
-from sdap_ingest_manager.config.ConfigMap import ConfigMap
-from sdap_ingest_manager.config.RemoteGitConfig import RemoteGitConfig
-
-
-class ConfigMapTest(unittest.TestCase):
- def test_createconfigmap(self):
-
- remote_git_config = RemoteGitConfig("https://github.com/tloubrieu-jpl/sdap-ingester-config")
-
- config_map = ConfigMap('collection-ingester', 'sdap', remote_git_config)
- config_map.publish()
-
-
-
-if __name__ == '__main__':
- unittest.main()