You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by al...@apache.org on 2018/08/08 13:58:00 UTC
nifi git commit: NIFI-5487 Move the packages to a version agnostic
path and create symlinks in place, add tests to cover,
add missing docs on nifi-toolkit
Repository: nifi
Updated Branches:
refs/heads/master 57baae9ae -> 9a79c94f8
NIFI-5487 Move the packages to a version agnostic path and create symlinks in place, add tests to cover, add missing docs on nifi-toolkit
This closes #2940.
Signed-off-by: Aldrin Piri <al...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/9a79c94f
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/9a79c94f
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/9a79c94f
Branch: refs/heads/master
Commit: 9a79c94f809a29ebb5c36ccca8a680f1875af1f2
Parents: 57baae9
Author: pepov <pe...@gmail.com>
Authored: Wed Aug 8 14:03:10 2018 +0200
Committer: Aldrin Piri <al...@apache.org>
Committed: Wed Aug 8 09:57:22 2018 -0400
----------------------------------------------------------------------
nifi-docker/dockerhub/Dockerfile | 26 ++++++++-----
nifi-docker/dockerhub/README.md | 48 ++++++++++++++++++++----
nifi-docker/dockermaven/Dockerfile | 16 +++++---
nifi-docker/dockermaven/integration-test.sh | 14 +++++--
4 files changed, 76 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/9a79c94f/nifi-docker/dockerhub/Dockerfile
----------------------------------------------------------------------
diff --git a/nifi-docker/dockerhub/Dockerfile b/nifi-docker/dockerhub/Dockerfile
index c3df2f6..7c844ce 100644
--- a/nifi-docker/dockerhub/Dockerfile
+++ b/nifi-docker/dockerhub/Dockerfile
@@ -29,7 +29,8 @@ ARG NIFI_BINARY_PATH=${NIFI_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-${NIFI_VERSI
ARG NIFI_TOOLKIT_BINARY_PATH=${NIFI_TOOLKIT_BINARY_PATH:-/nifi/${NIFI_VERSION}/nifi-toolkit-${NIFI_VERSION}-bin.zip}
ENV NIFI_BASE_DIR=/opt/nifi
-ENV NIFI_HOME=${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}
+ENV NIFI_HOME ${NIFI_BASE_DIR}/nifi-current
+ENV NIFI_TOOLKIT_HOME ${NIFI_BASE_DIR}/nifi-toolkit-current
ENV NIFI_PID_DIR=${NIFI_HOME}/run
ENV NIFI_LOG_DIR=${NIFI_HOME}/logs
@@ -39,13 +40,7 @@ ADD sh/ ${NIFI_BASE_DIR}/scripts/
# Setup NiFi user and create necessary directories
RUN groupadd -g ${GID} nifi || groupmod -n nifi `getent group ${GID} | cut -d: -f1` \
&& useradd --shell /bin/bash -u ${UID} -g ${GID} -m nifi \
- && mkdir -p ${NIFI_HOME}/conf \
- && mkdir -p ${NIFI_HOME}/database_repository \
- && mkdir -p ${NIFI_HOME}/flowfile_repository \
- && mkdir -p ${NIFI_HOME}/content_repository \
- && mkdir -p ${NIFI_HOME}/provenance_repository \
- && mkdir -p ${NIFI_HOME}/state \
- && mkdir -p ${NIFI_LOG_DIR} \
+ && mkdir -p ${NIFI_BASE_DIR} \
&& chown -R nifi:nifi ${NIFI_BASE_DIR} \
&& apt-get update \
&& apt-get install -y jq xmlstarlet procps
@@ -56,13 +51,24 @@ USER nifi
RUN curl -fSL ${MIRROR_BASE_URL}/${NIFI_TOOLKIT_BINARY_PATH} -o ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip \
&& echo "$(curl ${BASE_URL}/${NIFI_TOOLKIT_BINARY_PATH}.sha256) *${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip" | sha256sum -c - \
&& unzip ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip -d ${NIFI_BASE_DIR} \
- && rm ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip
+ && rm ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip \
+ && mv ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION} ${NIFI_TOOLKIT_HOME} \
+ && ln -s ${NIFI_TOOLKIT_HOME} ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}
# Download, validate, and expand Apache NiFi binary.
RUN curl -fSL ${MIRROR_BASE_URL}/${NIFI_BINARY_PATH} -o ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip \
&& echo "$(curl ${BASE_URL}/${NIFI_BINARY_PATH}.sha256) *${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip" | sha256sum -c - \
&& unzip ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip -d ${NIFI_BASE_DIR} \
- && rm ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip
+ && rm ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip \
+ && mv ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION} ${NIFI_HOME} \
+ && mkdir -p ${NIFI_HOME}/conf \
+ && mkdir -p ${NIFI_HOME}/database_repository \
+ && mkdir -p ${NIFI_HOME}/flowfile_repository \
+ && mkdir -p ${NIFI_HOME}/content_repository \
+ && mkdir -p ${NIFI_HOME}/provenance_repository \
+ && mkdir -p ${NIFI_HOME}/state \
+ && mkdir -p ${NIFI_LOG_DIR} \
+ && ln -s ${NIFI_HOME} ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}
VOLUME ${NIFI_LOG_DIR} \
${NIFI_HOME}/conf \
http://git-wip-us.apache.org/repos/asf/nifi/blob/9a79c94f/nifi-docker/dockerhub/README.md
----------------------------------------------------------------------
diff --git a/nifi-docker/dockerhub/README.md b/nifi-docker/dockerhub/README.md
index 5c523b0..dabf7eb 100644
--- a/nifi-docker/dockerhub/README.md
+++ b/nifi-docker/dockerhub/README.md
@@ -13,13 +13,33 @@
limitations under the License.
-->
+## Latest changes
+
+### 1.8.0
+- The NiFi Toolkit has been added to the image under the path `/opt/nifi/nifi-toolkit-current` also set as the environment variable `NIFI_TOOLKIT_HOME`
+- The installation directory and related environment variables are changed to be version-agnostic to `/opt/nifi/nifi-current`:
+```
+docker run --rm --entrypoint /bin/bash apache/nifi:1.8.0 -c 'env | grep NIFI'
+NIFI_HOME=/opt/nifi/nifi-current
+NIFI_LOG_DIR=/opt/nifi/nifi-current/logs
+NIFI_TOOLKIT_HOME=/opt/nifi/nifi-toolkit-current
+NIFI_PID_DIR=/opt/nifi/nifi-current/run
+NIFI_BASE_DIR=/opt/nifi
+```
+- A symlink refer to the new path for backward compatibility:
+```
+docker run --rm --entrypoint /bin/bash apache/nifi:1.8.0 -c 'readlink /opt/nifi/nifi-1.8.0' /opt/nifi/nifi-current
+```
+
# Docker Image Quickstart
## Capabilities
This image currently supports running in standalone mode either unsecured or with user authentication provided through:
* [Two-Way SSL with Client Certificates](http://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#security-configuration)
* [Lightweight Directory Access Protocol (LDAP)](http://nifi.apache.org/docs/nifi-docs/html/administration-guide.html#ldap_login_identity_provider)
-
+
+This image also contains the NiFi Toolkit (as of version 1.8.0) preconfigured to use either in secure and unsecure mode.
+
## Building
The Docker image can be built using the following command:
@@ -34,7 +54,7 @@ This build will result in an image tagged apache/nifi:latest
**Note**: The default version of NiFi specified by the Dockerfile is typically that of one that is unreleased if working from source.
To build an image for a prior released version, one can override the `NIFI_VERSION` build-arg with the following command:
-
+
docker build --build-arg=NIFI_VERSION={Desired NiFi Version} -t apache/nifi:latest .
There is, however, no guarantee that older versions will work as properties have changed and evolved with subsequent releases.
@@ -49,7 +69,7 @@ The minimum to run a NiFi instance is as follows:
-p 8080:8080 \
-d \
apache/nifi:latest
-
+
This will provide a running instance, exposing the instance UI to the host system on at port 8080,
viewable at `http://localhost:8080/nifi`.
@@ -60,9 +80,9 @@ You can also pass in environment variables to change the NiFi communication port
-d \
-e NIFI_WEB_HTTP_PORT='9090'
apache/nifi:latest
-
+
For a list of the environment variables recognised in this build, look into the .sh/secure.sh and .sh/start.sh scripts
-
+
### Standalone Instance, Two-Way SSL
In this configuration, the user will need to provide certificates and the associated configuration information.
Of particular note, is the `AUTH` environment variable which is set to `tls`. Additionally, the user must provide an
@@ -88,8 +108,8 @@ Finally, this command makes use of a volume to provide certificates on the host
In this configuration, the user will need to provide certificates and the associated configuration information. Optionally,
if the LDAP provider of interest is operating in LDAPS or START_TLS modes, certificates will additionally be needed.
Of particular note, is the `AUTH` environment variable which is set to `ldap`. Additionally, the user must provide a
-DN as provided by the configured LDAP server in the `INITIAL_ADMIN_IDENTITY` environment variable. This value will be
-used to seed the instance with an initial user with administrative privileges. Finally, this command makes use of a
+DN as provided by the configured LDAP server in the `INITIAL_ADMIN_IDENTITY` environment variable. This value will be
+used to seed the instance with an initial user with administrative privileges. Finally, this command makes use of a
volume to provide certificates on the host system to the container instance.
#### For a minimal, connection to an LDAP server using SIMPLE authentication:
@@ -147,8 +167,20 @@ volume to provide certificates on the host system to the container instance.
| Connect String | NIFI_ZK_CONNECT_STRING |
| Root Node | NIFI_ZK_ROOT_NODE |
+
+### Using the Toolkit
+
+Start the container:
+
+ docker run -d --name nifi apache/nifi
+
+After NiFi has been started, it is possible to run toolkit commands against the running instance:
+
+ docker exec -ti nifi nifi-toolkit-current/bin/cli.sh nifi current-user
+ anonymous
+
## Configuration Information
-The following ports are specified by default in Docker for NiFi operation within the container and
+The following ports are specified by default in Docker for NiFi operation within the container and
can be published to the host.
| Function | Property | Port |
http://git-wip-us.apache.org/repos/asf/nifi/blob/9a79c94f/nifi-docker/dockermaven/Dockerfile
----------------------------------------------------------------------
diff --git a/nifi-docker/dockermaven/Dockerfile b/nifi-docker/dockermaven/Dockerfile
index 796a8fb..25c3ab7 100644
--- a/nifi-docker/dockermaven/Dockerfile
+++ b/nifi-docker/dockermaven/Dockerfile
@@ -24,7 +24,8 @@ ARG NIFI_BINARY
ARG NIFI_TOOLKIT_BINARY
ENV NIFI_BASE_DIR /opt/nifi
-ENV NIFI_HOME $NIFI_BASE_DIR/nifi-$NIFI_VERSION
+ENV NIFI_HOME ${NIFI_BASE_DIR}/nifi-current
+ENV NIFI_TOOLKIT_HOME ${NIFI_BASE_DIR}/nifi-toolkit-current
ENV NIFI_PID_DIR=${NIFI_HOME}/run
ENV NIFI_LOG_DIR=${NIFI_HOME}/logs
@@ -32,11 +33,15 @@ ADD sh/ ${NIFI_BASE_DIR}/scripts/
COPY $NIFI_BINARY $NIFI_BASE_DIR
RUN unzip ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip -d ${NIFI_BASE_DIR} \
- && rm ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip
+ && rm ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip \
+ && mv ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION} ${NIFI_HOME} \
+ && ln -s ${NIFI_HOME} ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}
COPY $NIFI_TOOLKIT_BINARY $NIFI_BASE_DIR
RUN unzip ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip -d ${NIFI_BASE_DIR} \
- && rm ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip
+ && rm ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}-bin.zip \
+ && mv ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION} ${NIFI_TOOLKIT_HOME} \
+ && ln -s ${NIFI_TOOLKIT_HOME} ${NIFI_BASE_DIR}/nifi-toolkit-${NIFI_VERSION}
# Create necessary directories
RUN mkdir -p ${NIFI_HOME}/conf \
@@ -53,10 +58,9 @@ LABEL maintainer="Apache NiFi <de...@nifi.apache.org>"
ARG UID=1000
ARG GID=1000
-ARG NIFI_VERSION
-
ENV NIFI_BASE_DIR /opt/nifi
-ENV NIFI_HOME $NIFI_BASE_DIR/nifi-$NIFI_VERSION
+ENV NIFI_HOME ${NIFI_BASE_DIR}/nifi-current
+ENV NIFI_TOOLKIT_HOME ${NIFI_BASE_DIR}/nifi-toolkit-current
ENV NIFI_PID_DIR=${NIFI_HOME}/run
ENV NIFI_LOG_DIR=${NIFI_HOME}/logs
http://git-wip-us.apache.org/repos/asf/nifi/blob/9a79c94f/nifi-docker/dockermaven/integration-test.sh
----------------------------------------------------------------------
diff --git a/nifi-docker/dockermaven/integration-test.sh b/nifi-docker/dockermaven/integration-test.sh
index e1eedda..4b3b2fb 100755
--- a/nifi-docker/dockermaven/integration-test.sh
+++ b/nifi-docker/dockermaven/integration-test.sh
@@ -20,15 +20,18 @@ set -exuo pipefail
TAG=$1
VERSION=$2
-trap "{ docker rm -f nifi-${TAG}-integration-test; }" EXIT
+trap "{ docker ps -qaf Name=nifi-${TAG}-integration-test | xargs docker rm -f; }" EXIT
echo "Checking that all files are owned by NiFi"
test -z $(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c "find /opt/nifi ! -user nifi")
echo "Checking environment variables"
-test "/opt/nifi/nifi-${VERSION}" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_HOME')"
-test "/opt/nifi/nifi-${VERSION}/logs" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_LOG_DIR')"
-test "/opt/nifi/nifi-${VERSION}/run" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_PID_DIR')"
+test "/opt/nifi/nifi-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_HOME')"
+test "/opt/nifi/nifi-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c "readlink \${NIFI_BASE_DIR}/nifi-${VERSION}")"
+test "/opt/nifi/nifi-toolkit-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c "readlink \${NIFI_BASE_DIR}/nifi-toolkit-${VERSION}")"
+
+test "/opt/nifi/nifi-current/logs" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_LOG_DIR')"
+test "/opt/nifi/nifi-current/run" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_PID_DIR')"
test "/opt/nifi" = "$(docker run --rm --entrypoint /bin/bash apache/nifi:${TAG} -c 'echo -n $NIFI_BASE_DIR')"
echo "Starting NiFi container..."
@@ -46,5 +49,8 @@ done
echo "Checking system diagnostics"
test ${VERSION} = $(docker exec nifi-${TAG}-integration-test bash -c "curl -s $IP:8080/nifi-api/system-diagnostics | jq .systemDiagnostics.aggregateSnapshot.versionInfo.niFiVersion -r")
+echo "Checking current user with nifi-toolkit cli"
+test "anonymous" = $(docker exec nifi-${TAG}-integration-test bash -c '$NIFI_TOOLKIT_HOME/bin/cli.sh nifi current-user')
+
echo "Stopping NiFi container"
time docker stop nifi-${TAG}-integration-test
\ No newline at end of file