You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2020/09/28 16:06:34 UTC
[ranger] branch master updated: RANGER-3012: Dockerfile - updated
default to build from local repo (instead of from github)
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new b444fd9 RANGER-3012: Dockerfile - updated default to build from local repo (instead of from github)
b444fd9 is described below
commit b444fd9928fd6a7178214ce592b0f10aaf8936ad
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Thu Sep 24 12:24:47 2020 -0700
RANGER-3012: Dockerfile - updated default to build from local repo (instead of from github)
---
dev-support/ranger-docker/.env | 11 ++++
dev-support/ranger-docker/Dockerfile.ranger | 1 +
dev-support/ranger-docker/Dockerfile.ranger-base | 6 ---
dev-support/ranger-docker/Dockerfile.ranger-build | 9 ++--
dev-support/ranger-docker/Dockerfile.ranger-hadoop | 3 ++
dev-support/ranger-docker/Dockerfile.ranger-hbase | 3 ++
dev-support/ranger-docker/Dockerfile.ranger-kafka | 3 ++
dev-support/ranger-docker/README.md | 62 +++++++++++-----------
.../ranger-docker/docker-compose.ranger-base.yml | 2 +
.../ranger-docker/docker-compose.ranger-build.yml | 12 +++--
.../ranger-docker/docker-compose.ranger-hadoop.yml | 6 +++
.../ranger-docker/docker-compose.ranger-hbase.yml | 6 +++
.../ranger-docker/docker-compose.ranger-kafka.yml | 6 +++
.../ranger-docker/docker-compose.ranger.yml | 4 ++
dev-support/ranger-docker/patches/.gitignore | 1 +
dev-support/ranger-docker/scripts/ranger-build.sh | 51 ++++++++++++++++--
16 files changed, 138 insertions(+), 48 deletions(-)
diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env
new file mode 100644
index 0000000..a3e40d9
--- /dev/null
+++ b/dev-support/ranger-docker/.env
@@ -0,0 +1,11 @@
+BUILD_HOST_SRC=true
+SKIPTESTS=true
+GIT_URL=https://github.com/apache/ranger.git
+BRANCH=master
+
+POSTGRES_PASSWORD=rangerR0cks!
+
+RANGER_VERSION=3.0.0-SNAPSHOT
+HADOOP_VERSION=3.1.1
+HBASE_VERSION=2.0.3
+KAFKA_VERSION=2.4.0
diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger
index d414592..6225595 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger
+++ b/dev-support/ranger-docker/Dockerfile.ranger
@@ -16,6 +16,7 @@
FROM ranger-base:latest
+ARG RANGER_VERSION
COPY ./dist/version ${RANGER_DIST}/
COPY ./scripts/ranger.sh ${RANGER_SCRIPTS}/
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-base b/dev-support/ranger-docker/Dockerfile.ranger-base
index 383de3d..a09fbda 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-base
+++ b/dev-support/ranger-docker/Dockerfile.ranger-base
@@ -17,12 +17,6 @@
FROM ubuntu:20.04
-ENV RANGER_VERSION 3.0.0-SNAPSHOT
-ENV HADOOP_VERSION 3.1.1
-ENV HIVE_VERSION 3.1.2
-ENV HBASE_VERSION 2.0.3
-ENV KAFKA_VERSION 2.4.0
-
# Install curl, wget, tzdata, Python, Java, python-requests
RUN apt-get update && \
DEBIAN_FRONTEND="noninteractive" apt-get -y install curl wget tzdata \
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-build b/dev-support/ranger-docker/Dockerfile.ranger-build
index 56c6d5d..ef9b002 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-build
+++ b/dev-support/ranger-docker/Dockerfile.ranger-build
@@ -31,13 +31,12 @@ RUN mkdir -p /home/ranger/git && \
COPY ./scripts/ranger-build.sh /home/ranger/scripts/
-VOLUME /home/ranger/dist
VOLUME /home/ranger/.m2
+VOLUME /home/ranger/scripts
+VOLUME /home/ranger/patches
+VOLUME /home/ranger/dist
+VOLUME /home/ranger/src
USER ranger
-WORKDIR /home/ranger/git
-
-RUN git clone https://github.com/apache/ranger.git
-
ENTRYPOINT [ "/home/ranger/scripts/ranger-build.sh" ]
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hadoop b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
index 6f1e89f..c9513b6 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-hadoop
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hadoop
@@ -16,6 +16,9 @@
FROM ranger-base:latest
+ARG HADOOP_VERSION
+ARG RANGER_VERSION
+
COPY ./dist/version /home/ranger/dist/
COPY ./dist/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz /home/ranger/dist/
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-hbase b/dev-support/ranger-docker/Dockerfile.ranger-hbase
index a995250..5dea37a 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-hbase
+++ b/dev-support/ranger-docker/Dockerfile.ranger-hbase
@@ -16,6 +16,9 @@
FROM ranger-base:latest
+ARG HBASE_VERSION
+ARG RANGER_VERSION
+
COPY ./dist/version /home/ranger/dist/
COPY ./dist/ranger-${RANGER_VERSION}-hbase-plugin.tar.gz /home/ranger/dist/
diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kafka b/dev-support/ranger-docker/Dockerfile.ranger-kafka
index 42fb90f..72610d5 100644
--- a/dev-support/ranger-docker/Dockerfile.ranger-kafka
+++ b/dev-support/ranger-docker/Dockerfile.ranger-kafka
@@ -16,6 +16,9 @@
FROM ranger-base:latest
+ARG KAFKA_VERSION
+ARG RANGER_VERSION
+
COPY ./dist/version /home/ranger/dist/
COPY ./dist/ranger-${RANGER_VERSION}-kafka-plugin.tar.gz /home/ranger/dist/
diff --git a/dev-support/ranger-docker/README.md b/dev-support/ranger-docker/README.md
index 3f82db6..190b6db 100644
--- a/dev-support/ranger-docker/README.md
+++ b/dev-support/ranger-docker/README.md
@@ -27,94 +27,96 @@ Docker files in this folder create docker images and run them to build Apache Ra
2. Set this folder as your working directory.
-3. Using docker-compose is the simpler way to build and deploy Apache Ranger in containers.
+3. Update environment variables in .env file, if necessary
- 3.1. Execute following command to build Apache Ranger:
+4. Using docker-compose is the simpler way to build and deploy Apache Ranger in containers.
- docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger-build.yml up --remove-orphans
+ 4.1. Execute following command to build Apache Ranger:
+
+ docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger-build.yml up
Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
- 3.2. Execute following command to start Ranger, Ranger enabled HDFS/YARN/HBase/Kafka and dependent services (Solr, DB) in containers:
+ 4.2. Execute following command to start Ranger, Ranger enabled HDFS/YARN/HBase/Kafka and dependent services (Solr, DB) in containers:
docker-compose -f docker-compose.ranger-base.yml -f docker-compose.ranger.yml -f docker-compose.ranger-hadoop.yml -f docker-compose.ranger-hbase.yml -f docker-compose.ranger-kafka.yml up -d
-4. Alternatively docker command can be used to build and deploy Apache Ranger.
+5. Alternatively docker command can be used to build and deploy Apache Ranger.
- 4.1. Execute following command to build Docker image **ranger-base**:
+ 5.1. Execute following command to build Docker image **ranger-base**:
docker build -f Dockerfile.ranger-base -t ranger-base .
This might take about 10 minutes to complete.
- 4.2. Execute following command to build Docker image **ranger-build**:
+ 5.2. Execute following command to build Docker image **ranger-build**:
docker build -f Dockerfile.ranger-build -t ranger-build .
- 4.3. Build Apache Ranger in a container with the following command:
+ 5.3. Build Apache Ranger in a container with the following command:
- docker run -it --rm -v ${HOME}/.m2:/home/ranger/.m2 -v $(pwd)/dist:/home/ranger/dist -e BRANCH=ranger-2.1 -e PROFILE=all -e SKIPTESTS=true ranger-build
+ docker run -it --rm -v ${HOME}/.m2:/home/ranger/.m2:delegated -v $(pwd)/scripts:/home/ranger/scripts -v $(pwd)/../..:/home/ranger/src:delegated -v $(pwd)/dist:/home/ranger/dist --env-file ./.env ranger-build
Time taken to complete the build might vary (upto an hour), depending on status of ${HOME}/.m2 directory cache.
- 4.4. Execute following command to build Docker image **ranger**:
+ 5.4. Execute following command to build Docker image **ranger**:
- docker build -f Dockerfile.ranger -t ranger .
+ docker build -f Dockerfile.ranger --build-arg RANGER_VERSION=`cat dist/version` -t ranger .
This might take about 10 minutes to complete.
- 4.5. Execute following command to build a Docker image **ranger-solr**:
+ 5.5. Execute following command to build a Docker image **ranger-solr**:
docker build -f Dockerfile.ranger-solr -t ranger-solr .
- 4.6. Execute following command to start a container that runs database for use by Ranger Admin:
+ 5.6. Execute following command to start a container that runs database for use by Ranger Admin:
- docker run --name ranger-db --hostname ranger-db.example.com -e POSTGRES_PASSWORD='rangerR0cks!' -d postgres:12
+ docker run --name ranger-db --hostname ranger-db.example.com --env-file ./.env -d postgres:12
- 4.7. Execute following command to start a container that runs Solr for use by Ranger Admin:
+ 5.7. Execute following command to start a container that runs Solr for use by Ranger Admin:
docker run --name ranger-solr --hostname ranger-solr.example.com -p 8983:8983 -d ranger-solr solr-precreate ranger_audits /opt/solr/server/solr/configsets/ranger_audits/
- 4.8. Execute following command to install and run Ranger services in a container:
+ 5.8. Execute following command to install and run Ranger services in a container:
- docker run -it -d --name ranger --hostname ranger.example.com -p 6080:6080 --link ranger-db:ranger-db --link ranger-solr:ranger-solr ranger
+ docker run -it -d --name ranger --hostname ranger.example.com -p 6080:6080 --link ranger-db:ranger-db --link ranger-solr:ranger-solr --env-file ./.env ranger
This might take few minutes to complete.
- 4.9. Execute following command to build Docker image **ranger-hadoop**:
+ 5.9. Execute following command to build Docker image **ranger-hadoop**:
- docker build -f Dockerfile.ranger-hadoop -t ranger-hadoop .
+ docker build -f Dockerfile.ranger-hadoop --build-arg RANGER_VERSION=`cat dist/version` --build-arg HADOOP_VERSION=3.1.1 -t ranger-hadoop .
This step includes downloading of Hadoop tar balls, and can take a while to complete.
- 4.10. Execute following command to install and run Ranger enabled HDFS in a container:
+ 5.10. Execute following command to install and run Ranger enabled HDFS in a container:
- docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 -p 8088:8088 --link ranger:ranger --link ranger-solr:ranger-solr ranger-hadoop
+ docker run -it -d --name ranger-hadoop --hostname ranger-hadoop.example.com -p 9000:9000 -p 8088:8088 --link ranger:ranger --link ranger-solr:ranger-solr --env-file ./.env ranger-hadoop
This might take few minutes to complete.
- 4.11. Execute following command to build Docker image **ranger-hbase**:
+ 5.11. Execute following command to build Docker image **ranger-hbase**:
- docker build -f Dockerfile.ranger-hbase -t ranger-hbase .
+ docker build -f Dockerfile.ranger-hbase --build-arg RANGER_VERSION=`cat dist/version` --build-arg HBASE_VERSION=2.0.3 -t ranger-hbase .
This step includes downloading of HBase tar ball, and can take a while to complete.
- 4.12. Execute following command to install and run Ranger enabled HBase in a container:
+ 5.12. Execute following command to install and run Ranger enabled HBase in a container:
- docker run -it -d --name ranger-hbase --hostname ranger-hbase.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr ranger-hbase
+ docker run -it -d --name ranger-hbase --hostname ranger-hbase.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr --env-file ./.env ranger-hbase
This might take few minutes to complete.
- 4.13. Execute following command to build Docker image **ranger-kafka**:
+ 5.13. Execute following command to build Docker image **ranger-kafka**:
- docker build -f Dockerfile.ranger-kafka -t ranger-kafka .
+ docker build -f Dockerfile.ranger-kafka --build-arg RANGER_VERSION=`cat dist/version` --build-arg KAFKA_VERSION=2.4.0 -t ranger-kafka .
This step includes downloading of Kafka tar ball, and can take a while to complete.
- 4.12. Execute following command to install and run Ranger enabled Kafka in a container:
+ 5.14. Execute following command to install and run Ranger enabled Kafka in a container:
- docker run -it -d --name ranger-kafka --hostname ranger-kafka.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr ranger-kafka
+ docker run -it -d --name ranger-kafka --hostname ranger-kafka.example.com --link ranger-hadoop:ranger-hadoop --link ranger:ranger --link ranger-solr:ranger-solr --env-file ./.env ranger-kafka
This might take few minutes to complete.
-5. Ranger Admin can be accessed at http://localhost:6080 (admin/rangerR0cks!)
+6. Ranger Admin can be accessed at http://localhost:6080 (admin/rangerR0cks!)
diff --git a/dev-support/ranger-docker/docker-compose.ranger-base.yml b/dev-support/ranger-docker/docker-compose.ranger-base.yml
index 18e78db..614c5d3 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-base.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-base.yml
@@ -7,6 +7,8 @@ services:
image: ranger-base
networks:
- ranger
+ environment:
+ - RANGER_VERSION
networks:
ranger:
diff --git a/dev-support/ranger-docker/docker-compose.ranger-build.yml b/dev-support/ranger-docker/docker-compose.ranger-build.yml
index 9572be1..50ef383 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-build.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-build.yml
@@ -10,13 +10,19 @@ services:
networks:
- ranger
volumes:
- - ~/.m2:/home/ranger/.m2
+ - ~/.m2:/home/ranger/.m2:delegated
+ - ./scripts:/home/ranger/scripts
+ - ./patches:/home/ranger/patches
- ./dist:/home/ranger/dist
+ - ./../../:/home/ranger/src:delegated
depends_on:
- ranger-base
environment:
- BRANCH: 'master'
- SKIPTESTS: 'true'
+ - BRANCH
+ - BUILD_HOST_SRC
+ - GIT_URL
+ - RANGER_VERSION
+ - SKIPTESTS
networks:
ranger:
diff --git a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml
index e521345..54e5cb5 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-hadoop.yml
@@ -4,6 +4,9 @@ services:
build:
context: .
dockerfile: Dockerfile.ranger-hadoop
+ args:
+ - HADOOP_VERSION=${HADOOP_VERSION}
+ - RANGER_VERSION=${RANGER_VERSION}
image: ranger-hadoop
container_name: ranger-hadoop
hostname: ranger-hadoop.example.com
@@ -16,6 +19,9 @@ services:
- "8088:8088"
depends_on:
- ranger
+ environment:
+ - HADOOP_VERSION
+ - RANGER_VERSION
networks:
ranger:
diff --git a/dev-support/ranger-docker/docker-compose.ranger-hbase.yml b/dev-support/ranger-docker/docker-compose.ranger-hbase.yml
index 81804aa..46a9fee 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-hbase.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-hbase.yml
@@ -4,6 +4,9 @@ services:
build:
context: .
dockerfile: Dockerfile.ranger-hbase
+ args:
+ - HBASE_VERSION=${HBASE_VERSION}
+ - RANGER_VERSION=${RANGER_VERSION}
image: ranger-hbase
container_name: ranger-hbase
hostname: ranger-hbase.example.com
@@ -17,6 +20,9 @@ services:
- "16020:16020"
depends_on:
- ranger-hadoop
+ environment:
+ - HBASE_VERSION
+ - RANGER_VERSION
networks:
ranger:
diff --git a/dev-support/ranger-docker/docker-compose.ranger-kafka.yml b/dev-support/ranger-docker/docker-compose.ranger-kafka.yml
index 1d14f1b..5cf0b9a 100644
--- a/dev-support/ranger-docker/docker-compose.ranger-kafka.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger-kafka.yml
@@ -4,6 +4,9 @@ services:
build:
context: .
dockerfile: Dockerfile.ranger-kafka
+ args:
+ - KAFKA_VERSION=${KAFKA_VERSION}
+ - RANGER_VERSION=${RANGER_VERSION}
image: ranger-kafka
container_name: ranger-kafka
hostname: ranger-kafka.example.com
@@ -13,6 +16,9 @@ services:
- ranger
depends_on:
- ranger
+ environment:
+ - KAFKA_VERSION
+ - RANGER_VERSION
networks:
ranger:
diff --git a/dev-support/ranger-docker/docker-compose.ranger.yml b/dev-support/ranger-docker/docker-compose.ranger.yml
index 787c2ef..089a51b 100644
--- a/dev-support/ranger-docker/docker-compose.ranger.yml
+++ b/dev-support/ranger-docker/docker-compose.ranger.yml
@@ -4,6 +4,8 @@ services:
build:
context: .
dockerfile: Dockerfile.ranger
+ args:
+ - RANGER_VERSION=${RANGER_VERSION}
image: ranger:latest
container_name: ranger
hostname: ranger.example.com
@@ -17,6 +19,8 @@ services:
- ranger-base
- ranger-db
- ranger-solr
+ environment:
+ - RANGER_VERSION
command:
- /home/ranger/scripts/ranger.sh
diff --git a/dev-support/ranger-docker/patches/.gitignore b/dev-support/ranger-docker/patches/.gitignore
new file mode 100644
index 0000000..72e8ffc
--- /dev/null
+++ b/dev-support/ranger-docker/patches/.gitignore
@@ -0,0 +1 @@
+*
diff --git a/dev-support/ranger-docker/scripts/ranger-build.sh b/dev-support/ranger-docker/scripts/ranger-build.sh
index 256b5a1..99227bc 100755
--- a/dev-support/ranger-docker/scripts/ranger-build.sh
+++ b/dev-support/ranger-docker/scripts/ranger-build.sh
@@ -16,14 +16,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+
if [ "${BRANCH}" == "" ]
then
BRANCH=master
fi
+if [ "${GIT_URL}" == "" ]
+then
+ GIT_URL=https://github.com/apache/ranger.git
+fi
+
if [ "${PROFILE}" != "" ]
then
- ARG_PROFILES="-P ${PROFILE}"
+ ARG_PROFILES="-P${PROFILE}"
fi
if [ "${SKIPTESTS}" == "" ]
@@ -33,13 +39,50 @@ else
ARG_SKIPTESTS="-DskipTests=${SKIPTESTS}"
fi
+if [ "${BUILD_HOST_SRC}" == "" ]
+then
+ BUILD_HOST_SRC=true
+fi
+
export MAVEN_OPTS="-Xms2g -Xmx2g"
export M2=/home/ranger/.m2
-cd /home/ranger/git/ranger
-git checkout ${BRANCH}
-git pull
+if [ "${BUILD_HOST_SRC}" == "true" ]
+then
+ if [ ! -f /home/ranger/src/pom.xml ]
+ then
+ echo "ERROR: BUILD_HOST_SRC=${BUILD_HOST_SRC}, but /home/ranger/src/pom.xml is not found "
+ exit 1
+ fi
+
+ echo "Building from /home/ranger/src"
+
+ cd /home/ranger/src
+else
+ echo "Building ${BRANCH} branch from ${GIT_URL}"
+
+ cd /home/ranger/git
+
+ if [ -d ranger ]
+ then
+ renamedDir=ranger-`date +"%Y%m%d-%H%M%S"`
+
+ echo "Renaming existing directory `pwd`/ranger to ${renamedDir}"
+
+ mv ranger $renamedDir
+ fi
+
+ git clone --single-branch --branch ${BRANCH} ${GIT_URL}
+
+ cd /home/ranger/git/ranger
+
+ for patch in `ls -1 /home/ranger/patches | sort`
+ do
+ echo "applying patch /home/ranger/patches/${patch}"
+ git apply /home/ranger/patches/${patch}
+ done
+fi
mvn ${ARG_PROFILES} ${ARG_SKIPTESTS} -DskipDocs clean package