You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/06/22 23:56:43 UTC
[incubator-sdap-ingester] branch dockerize_collection_ingester
updated (32524d7 -> 2ef424a)
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a change to branch dockerize_collection_ingester
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git.
discard 32524d7 args
discard 213ee5c dockerfile
omit e0a23c4 SDAP-254, SDAP-255, SDAP-256: Fix bug where ingestion history is not saved, fix bug where messages published to RabbitMQ are incorrect, fix bug where bad collection config file crashes app
add 5850e16 SDAP-254, SDAP-255, SDAP-256: Fix bug where ingestion history is not saved, fix bug where messages published to RabbitMQ are incorrect, fix bug where bad collection config file crashes app (#3)
new 70608e1 dockerfile
new 2ef424a args
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (32524d7)
\
N -- N -- N refs/heads/dockerize_collection_ingester (2ef424a)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
[incubator-sdap-ingester] 02/02: args
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch dockerize_collection_ingester
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 2ef424a225492c9ff713e05c7e1048c1e9321eb0
Author: Eamon Ford <ea...@jpl.nasa.gov>
AuthorDate: Mon Jun 22 14:00:11 2020 -0700
args
---
collection_manager/collection_manager/main.py | 18 ++++++++----------
1 file changed, 8 insertions(+), 10 deletions(-)
diff --git a/collection_manager/collection_manager/main.py b/collection_manager/collection_manager/main.py
index bc2d356..72c9e84 100644
--- a/collection_manager/collection_manager/main.py
+++ b/collection_manager/collection_manager/main.py
@@ -8,7 +8,7 @@ from collection_manager.services.history_manager import SolrIngestionHistoryBuil
logging.basicConfig(level=logging.INFO)
logging.getLogger("pika").setLevel(logging.WARNING)
-logger = logging.getLogger("collection_manager")
+logger = logging.getLogger()
def check_path(path) -> str:
@@ -19,25 +19,23 @@ def check_path(path) -> str:
def get_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Run ingestion for a list of collection ingestion streams")
- parser.add_argument("--refresh",
- help="refresh interval in seconds to check for new or updated granules",
- default=300)
- parser.add_argument("--collections",
+ parser.add_argument("--collections-path",
help="Absolute path to collections configuration file",
+ metavar="PATH",
required=True)
- parser.add_argument('--rabbitmq_host',
+ parser.add_argument('--rabbitmq-host',
default='localhost',
metavar='HOST',
help='RabbitMQ hostname to connect to. (Default: "localhost")')
- parser.add_argument('--rabbitmq_username',
+ parser.add_argument('--rabbitmq-username',
default='guest',
metavar='USERNAME',
help='RabbitMQ username. (Default: "guest")')
- parser.add_argument('--rabbitmq_password',
+ parser.add_argument('--rabbitmq-password',
default='guest',
metavar='PASSWORD',
help='RabbitMQ password. (Default: "guest")')
- parser.add_argument('--rabbitmq_queue',
+ parser.add_argument('--rabbitmq-queue',
default="nexus",
metavar="QUEUE",
help='Name of the RabbitMQ queue to consume from. (Default: "nexus")')
@@ -65,7 +63,7 @@ def main():
publisher.connect()
collection_processor = CollectionProcessor(message_publisher=publisher,
history_manager_builder=history_manager_builder)
- collection_watcher = CollectionWatcher(collections_path=options.collections,
+ collection_watcher = CollectionWatcher(collections_path=options.collections_path,
collection_updated_callback=collection_processor.process_collection,
granule_updated_callback=collection_processor.process_granule)
[incubator-sdap-ingester] 01/02: dockerfile
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch dockerize_collection_ingester
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 70608e1075f70ef1b6abc4c72a8a371d55c29ef9
Author: Eamon Ford <ea...@jpl.nasa.gov>
AuthorDate: Mon Jun 22 11:34:27 2020 -0700
dockerfile
---
collection_manager/containers/docker/Dockerfile | 11 ++++++++---
collection_manager/containers/docker/entrypoint.sh | 10 ++++++++++
collection_manager/setup.py | 8 +++-----
3 files changed, 21 insertions(+), 8 deletions(-)
diff --git a/collection_manager/containers/docker/Dockerfile b/collection_manager/containers/docker/Dockerfile
index 3ba8da7..fcd3aa3 100644
--- a/collection_manager/containers/docker/Dockerfile
+++ b/collection_manager/containers/docker/Dockerfile
@@ -1,11 +1,16 @@
FROM python:3
-# Add kubernetes client to create other pods (ingester)
RUN apt-get update && apt-get install -y apt-transport-https gnupg2
RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
RUN echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list
RUN apt-get update && apt-get install -y kubectl
-RUN pip install https://github.com/tloubrieu-jpl/incubator-sdap-nexus-ingestion-manager/releases/download/0.4.0+dev/sdap_ingest_manager-0.4.0+dev-py3-none-any.whl
+COPY /collection_manager /collection_manager/collection_manager
+COPY /setup.py /collection_manager/setup.py
+COPY /requirements.txt /collection_manager/requirements.txt
+COPY /README.md /collection_manager/README.md
+COPY /containers/docker/entrypoint.sh /entrypoint.sh
-CMD bash
+RUN cd /collection_manager && python setup.py install
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
diff --git a/collection_manager/containers/docker/entrypoint.sh b/collection_manager/containers/docker/entrypoint.sh
new file mode 100644
index 0000000..eb88f75
--- /dev/null
+++ b/collection_manager/containers/docker/entrypoint.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+python /collection_manager/collection_manager/main.py \
+ $([[ ! -z "$COLLECTIONS_PATH" ]] && echo --collections-path=$COLLECTIONS_PATH) \
+ $([[ ! -z "$RABBITMQ_HOST" ]] && echo --rabbitmq-host=$RABBITMQ_HOST) \
+ $([[ ! -z "$RABBITMQ_USERNAME" ]] && echo --rabbitmq-username=$RABBITMQ_USERNAME) \
+ $([[ ! -z "$RABBITMQ_PASSWORD" ]] && echo --rabbitmq-password=$RABBITMQ_PASSWORD) \
+ $([[ ! -z "$RABBITMQ_QUEUE" ]] && echo --rabbitmq-queue=$RABBITMQ_QUEUE) \
+ $([[ ! -z "$HISTORY_URL" ]] && echo --history-url=$HISTORY_URL) \
+ $([[ ! -z "$HISTORY_PATH" ]] && echo --history-path=$HISTORY_PATH)
diff --git a/collection_manager/setup.py b/collection_manager/setup.py
index 49b0d75..1542486 100644
--- a/collection_manager/setup.py
+++ b/collection_manager/setup.py
@@ -1,9 +1,7 @@
-import setuptools
-import os
-import subprocess
-import sys
import re
+import setuptools
+
PACKAGE_NAME = "sdap_collection_manager"
with open("./collection_manager/__init__.py") as fi:
@@ -24,7 +22,7 @@ setuptools.setup(
description="a helper to ingest data in sdap",
long_description=long_description,
long_description_content_type="text/markdown",
- url="https://github.com/tloubrieu-jpl/incubator-sdap-nexus-ingestion-manager",
+ url="https://github.com/apache/incubator-sdap-ingester",
packages=setuptools.find_packages(),
classifiers=[
"Programming Language :: Python :: 3",