You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2022/11/17 04:03:47 UTC
[atlas] 04/04: ATLAS-3820: fix import-kafka.sh failures; updated docker setup for Kafka bridge
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 2c2d9197e68b809198d6e9f1232473d1c2d9e960
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Tue Nov 15 14:22:04 2022 -0800
ATLAS-3820: fix import-kafka.sh failures; updated docker setup for Kafka bridge
---
addons/kafka-bridge/pom.xml | 72 +++++++++++++++++++++-
addons/kafka-bridge/src/bin/import-kafka.sh | 18 +-----
.../org/apache/atlas/kafka/bridge/KafkaBridge.java | 2 +-
dev-support/atlas-docker/.dockerignore | 1 +
dev-support/atlas-docker/Dockerfile.atlas-kafka | 21 ++++---
dev-support/atlas-docker/scripts/atlas-build.sh | 1 +
.../atlas-kafka-application.properties} | 24 ++------
7 files changed, 95 insertions(+), 44 deletions(-)
diff --git a/addons/kafka-bridge/pom.xml b/addons/kafka-bridge/pom.xml
index 9ce22a757..31f5d5b3a 100644
--- a/addons/kafka-bridge/pom.xml
+++ b/addons/kafka-bridge/pom.xml
@@ -125,7 +125,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>12.0.1</version>
+ <version>${guava.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -223,6 +223,21 @@
<artifactId>atlas-common</artifactId>
<version>${project.version}</version>
</artifactItem>
+ <artifactItem>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ <version>${commons-collections.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ <version>${commons-logging.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commons-lang.version}</version>
+ </artifactItem>
<artifactItem>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
@@ -233,11 +248,66 @@
<artifactId>jersey-multipart</artifactId>
<version>${jersey.version}</version>
</artifactItem>
+ <artifactItem>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.databind.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${jackson.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <version>${jackson.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-base</artifactId>
+ <version>${jackson.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-bundle</artifactId>
+ <version>${jersey.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${hadoop.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponents-httpclient.version}</version>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${httpcomponents-httpcore.version}</version>
+ </artifactItem>
<artifactItem>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>${commons-conf.version}</version>
</artifactItem>
+ <artifactItem>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>${json-simple.version}</version>
+ </artifactItem>
</artifactItems>
</configuration>
</execution>
diff --git a/addons/kafka-bridge/src/bin/import-kafka.sh b/addons/kafka-bridge/src/bin/import-kafka.sh
index ada3b4ae2..8fe27748a 100644
--- a/addons/kafka-bridge/src/bin/import-kafka.sh
+++ b/addons/kafka-bridge/src/bin/import-kafka.sh
@@ -99,25 +99,12 @@ fi
KAFKA_CP="${KAFKA_CONF}"
-for i in "${KAFKA_HOME}/libs/kafka-clients"*.jar "${KAFKA_HOME}/libs/slf4j-api"*.jar; do
+for i in "${KAFKA_HOME}/libs/kafka-clients"*.jar "${KAFKA_HOME}/libs/slf4j-"*.jar "${KAFKA_HOME}/libs/log4j-"*.jar "${KAFKA_HOME}/libs/commons-"*.jar "${KAFKA_HOME}/libs/jackson-module-jaxb-annotations"*.jar; do
KAFKA_CP="${KAFKA_CP}:$i"
done
-#Add hadoop conf in classpath
-if [ ! -z "$HADOOP_CLASSPATH" ]; then
- HADOOP_CP=$HADOOP_CLASSPATH
-elif [ ! -z "$HADOOP_HOME" ]; then
- HADOOP_CP=`$HADOOP_HOME/bin/hadoop classpath`
-elif [ $(command -v hadoop) ]; then
- HADOOP_CP=`hadoop classpath`
- #echo $HADOOP_CP
-else
- echo "Environment variable HADOOP_CLASSPATH or HADOOP_HOME need to be set"
- exit 1
-fi
-
-CP="${ATLASCPPATH}:${HADOOP_CP}:${KAFKA_CP}"
+CP="${ATLASCPPATH}:${KAFKA_CP}"
# If running in cygwin, convert pathnames and classpath to Windows format.
if [ "${CYGWIN}" == "true" ]
@@ -125,7 +112,6 @@ then
ATLAS_LOG_DIR=`cygpath -w ${ATLAS_LOG_DIR}`
LOGFILE=`cygpath -w ${LOGFILE}`
KAFKA_CP=`cygpath -w ${KAFKA_CP}`
- HADOOP_CP=`cygpath -w ${HADOOP_CP}`
CP=`cygpath -w -p ${CP}`
fi
diff --git a/addons/kafka-bridge/src/main/java/org/apache/atlas/kafka/bridge/KafkaBridge.java b/addons/kafka-bridge/src/main/java/org/apache/atlas/kafka/bridge/KafkaBridge.java
index 8be2fca3a..0878ec29f 100644
--- a/addons/kafka-bridge/src/main/java/org/apache/atlas/kafka/bridge/KafkaBridge.java
+++ b/addons/kafka-bridge/src/main/java/org/apache/atlas/kafka/bridge/KafkaBridge.java
@@ -134,7 +134,7 @@ public class KafkaBridge {
KafkaBridge importer = new KafkaBridge(atlasConf, atlasClientV2, kafkaUtils);
- if(!KAFKA_SCHEMA_REGISTRY_ENV_VARIABLE.isEmpty()){
+ if(StringUtils.isNotEmpty(KAFKA_SCHEMA_REGISTRY_ENV_VARIABLE)) {
KAFKA_SCHEMA_REGISTRY_HOSTNAME = KAFKA_SCHEMA_REGISTRY_ENV_VARIABLE;
}
diff --git a/dev-support/atlas-docker/.dockerignore b/dev-support/atlas-docker/.dockerignore
index d4a3db26b..14ff6a9fc 100644
--- a/dev-support/atlas-docker/.dockerignore
+++ b/dev-support/atlas-docker/.dockerignore
@@ -2,6 +2,7 @@
!config
!dist/apache-atlas-*-hbase-hook.tar.gz
!dist/apache-atlas-*-hive-hook.tar.gz
+!dist/apache-atlas-*-kafka-hook.tar.gz
!dist/apache-atlas-*-server.tar.gz
!downloads/*
!scripts/*
diff --git a/dev-support/atlas-docker/Dockerfile.atlas-kafka b/dev-support/atlas-docker/Dockerfile.atlas-kafka
index e06d62845..d7a8e3f07 100644
--- a/dev-support/atlas-docker/Dockerfile.atlas-kafka
+++ b/dev-support/atlas-docker/Dockerfile.atlas-kafka
@@ -20,17 +20,24 @@ ARG ATLAS_VERSION
ARG KAFKA_VERSION
-COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/atlas/dist/
+COPY ./dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz /home/atlas/dist/
+COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/atlas/dist/
-COPY ./scripts/atlas-kafka-setup.sh /home/atlas/scripts/
-COPY ./scripts/atlas-kafka.sh /home/atlas/scripts/
+COPY ./scripts/atlas-kafka-setup.sh /home/atlas/scripts/
+COPY ./scripts/atlas-kafka.sh /home/atlas/scripts/
+COPY ./scripts/atlas-kafka-application.properties /home/atlas/scripts/
RUN tar xvfz /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \
ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \
- rm -f /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz
-
-ENV KAFKA_HOME /opt/kafka
-ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin
+ rm -f /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz && \
+ tar xvfz /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz --directory=/opt/ && \
+ ln -s /opt/apache-atlas-kafka-hook-${ATLAS_VERSION} /opt/apache-atlas-kafka-hook && \
+ rm -f /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz && \
+ cp -f /home/atlas/scripts/atlas-kafka-application.properties /opt/kafka/config/atlas-application.properties
+
+ENV KAFKA_HOME /opt/kafka
+ENV KAFKA_CONF_DIR /opt/kafka/config
+ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin
EXPOSE 9092
diff --git a/dev-support/atlas-docker/scripts/atlas-build.sh b/dev-support/atlas-docker/scripts/atlas-build.sh
index e53ebbba9..592a055f7 100755
--- a/dev-support/atlas-docker/scripts/atlas-build.sh
+++ b/dev-support/atlas-docker/scripts/atlas-build.sh
@@ -89,3 +89,4 @@ mvn ${ARG_PROFILES} ${ARG_SKIPTESTS} -DskipDocs clean package
mv -f distro/target/apache-atlas-${ATLAS_VERSION}-server.tar.gz /home/atlas/dist/
mv -f distro/target/apache-atlas-${ATLAS_VERSION}-hive-hook.tar.gz /home/atlas/dist/
mv -f distro/target/apache-atlas-${ATLAS_VERSION}-hbase-hook.tar.gz /home/atlas/dist/
+mv -f distro/target/apache-atlas-${ATLAS_VERSION}-kafka-hook.tar.gz /home/atlas/dist/
diff --git a/dev-support/atlas-docker/Dockerfile.atlas-kafka b/dev-support/atlas-docker/scripts/atlas-kafka-application.properties
similarity index 53%
copy from dev-support/atlas-docker/Dockerfile.atlas-kafka
copy to dev-support/atlas-docker/scripts/atlas-kafka-application.properties
index e06d62845..268e5c8c4 100644
--- a/dev-support/atlas-docker/Dockerfile.atlas-kafka
+++ b/dev-support/atlas-docker/scripts/atlas-kafka-application.properties
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -13,25 +14,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+#
-FROM atlas-base:latest
-
-ARG ATLAS_VERSION
-ARG KAFKA_VERSION
-
-
-COPY ./downloads/kafka_2.12-${KAFKA_VERSION}.tgz /home/atlas/dist/
-
-COPY ./scripts/atlas-kafka-setup.sh /home/atlas/scripts/
-COPY ./scripts/atlas-kafka.sh /home/atlas/scripts/
-
-RUN tar xvfz /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz --directory=/opt/ && \
- ln -s /opt/kafka_2.12-${KAFKA_VERSION} /opt/kafka && \
- rm -f /home/atlas/dist/kafka_2.12-${KAFKA_VERSION}.tgz
-
-ENV KAFKA_HOME /opt/kafka
-ENV PATH /usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kafka/bin
-EXPOSE 9092
+atlas.rest.address=http://atlas.example.com:21000
-ENTRYPOINT [ "/home/atlas/scripts/atlas-kafka.sh" ]
+atlas.kafka.zookeeper.connect=atlas-zk.example.com:2181
+atlas.kafka.bootstrap.servers=atlas-kafka.example.com:9092