You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/06/22 23:58:05 UTC
[incubator-sdap-ingester] 01/01: SDAP-237 Dockerize Collection
Manager
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch dockerize_collection_ingester
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 375c53994f0302793a184d237f775e64166361bb
Author: Eamon Ford <ea...@jpl.nasa.gov>
AuthorDate: Mon Jun 22 11:34:27 2020 -0700
SDAP-237 Dockerize Collection Manager
---
collection_manager/collection_manager/main.py | 18 ++++++++----------
collection_manager/containers/docker/Dockerfile | 11 ++++++++---
collection_manager/containers/docker/entrypoint.sh | 10 ++++++++++
collection_manager/setup.py | 8 +++-----
4 files changed, 29 insertions(+), 18 deletions(-)
diff --git a/collection_manager/collection_manager/main.py b/collection_manager/collection_manager/main.py
index bc2d356..72c9e84 100644
--- a/collection_manager/collection_manager/main.py
+++ b/collection_manager/collection_manager/main.py
@@ -8,7 +8,7 @@ from collection_manager.services.history_manager import SolrIngestionHistoryBuil
logging.basicConfig(level=logging.INFO)
logging.getLogger("pika").setLevel(logging.WARNING)
-logger = logging.getLogger("collection_manager")
+logger = logging.getLogger()
def check_path(path) -> str:
@@ -19,25 +19,23 @@ def check_path(path) -> str:
def get_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Run ingestion for a list of collection ingestion streams")
- parser.add_argument("--refresh",
- help="refresh interval in seconds to check for new or updated granules",
- default=300)
- parser.add_argument("--collections",
+ parser.add_argument("--collections-path",
help="Absolute path to collections configuration file",
+ metavar="PATH",
required=True)
- parser.add_argument('--rabbitmq_host',
+ parser.add_argument('--rabbitmq-host',
default='localhost',
metavar='HOST',
help='RabbitMQ hostname to connect to. (Default: "localhost")')
- parser.add_argument('--rabbitmq_username',
+ parser.add_argument('--rabbitmq-username',
default='guest',
metavar='USERNAME',
help='RabbitMQ username. (Default: "guest")')
- parser.add_argument('--rabbitmq_password',
+ parser.add_argument('--rabbitmq-password',
default='guest',
metavar='PASSWORD',
help='RabbitMQ password. (Default: "guest")')
- parser.add_argument('--rabbitmq_queue',
+ parser.add_argument('--rabbitmq-queue',
default="nexus",
metavar="QUEUE",
help='Name of the RabbitMQ queue to consume from. (Default: "nexus")')
@@ -65,7 +63,7 @@ def main():
publisher.connect()
collection_processor = CollectionProcessor(message_publisher=publisher,
history_manager_builder=history_manager_builder)
- collection_watcher = CollectionWatcher(collections_path=options.collections,
+ collection_watcher = CollectionWatcher(collections_path=options.collections_path,
collection_updated_callback=collection_processor.process_collection,
granule_updated_callback=collection_processor.process_granule)
diff --git a/collection_manager/containers/docker/Dockerfile b/collection_manager/containers/docker/Dockerfile
index 3ba8da7..fcd3aa3 100644
--- a/collection_manager/containers/docker/Dockerfile
+++ b/collection_manager/containers/docker/Dockerfile
@@ -1,11 +1,16 @@
FROM python:3
-# Add kubernetes client to create other pods (ingester)
RUN apt-get update && apt-get install -y apt-transport-https gnupg2
RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
RUN echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list
RUN apt-get update && apt-get install -y kubectl
-RUN pip install https://github.com/tloubrieu-jpl/incubator-sdap-nexus-ingestion-manager/releases/download/0.4.0+dev/sdap_ingest_manager-0.4.0+dev-py3-none-any.whl
+COPY /collection_manager /collection_manager/collection_manager
+COPY /setup.py /collection_manager/setup.py
+COPY /requirements.txt /collection_manager/requirements.txt
+COPY /README.md /collection_manager/README.md
+COPY /containers/docker/entrypoint.sh /entrypoint.sh
-CMD bash
+RUN cd /collection_manager && python setup.py install
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
diff --git a/collection_manager/containers/docker/entrypoint.sh b/collection_manager/containers/docker/entrypoint.sh
new file mode 100644
index 0000000..eb88f75
--- /dev/null
+++ b/collection_manager/containers/docker/entrypoint.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+python /collection_manager/collection_manager/main.py \
+ $([[ ! -z "$COLLECTIONS_PATH" ]] && echo --collections-path=$COLLECTIONS_PATH) \
+ $([[ ! -z "$RABBITMQ_HOST" ]] && echo --rabbitmq-host=$RABBITMQ_HOST) \
+ $([[ ! -z "$RABBITMQ_USERNAME" ]] && echo --rabbitmq-username=$RABBITMQ_USERNAME) \
+ $([[ ! -z "$RABBITMQ_PASSWORD" ]] && echo --rabbitmq-password=$RABBITMQ_PASSWORD) \
+ $([[ ! -z "$RABBITMQ_QUEUE" ]] && echo --rabbitmq-queue=$RABBITMQ_QUEUE) \
+ $([[ ! -z "$HISTORY_URL" ]] && echo --history-url=$HISTORY_URL) \
+ $([[ ! -z "$HISTORY_PATH" ]] && echo --history-path=$HISTORY_PATH)
diff --git a/collection_manager/setup.py b/collection_manager/setup.py
index 49b0d75..1542486 100644
--- a/collection_manager/setup.py
+++ b/collection_manager/setup.py
@@ -1,9 +1,7 @@
-import setuptools
-import os
-import subprocess
-import sys
import re
+import setuptools
+
PACKAGE_NAME = "sdap_collection_manager"
with open("./collection_manager/__init__.py") as fi:
@@ -24,7 +22,7 @@ setuptools.setup(
description="a helper to ingest data in sdap",
long_description=long_description,
long_description_content_type="text/markdown",
- url="https://github.com/tloubrieu-jpl/incubator-sdap-nexus-ingestion-manager",
+ url="https://github.com/apache/incubator-sdap-ingester",
packages=setuptools.find_packages(),
classifiers=[
"Programming Language :: Python :: 3",