You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by go...@apache.org on 2021/07/22 10:36:10 UTC
[incubator-inlong] branch master updated: [INLONG-786] add
dataproxy docker image (#580)
This is an automated email from the ASF dual-hosted git repository.
gosonzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 4b0692c [INLONG-786] add dataproxy docker image (#580)
4b0692c is described below
commit 4b0692c22bbc5bf2b0770b9c391c2034c19d8f6c
Author: dockerzhang <do...@apache.org>
AuthorDate: Thu Jul 22 18:36:01 2021 +0800
[INLONG-786] add dataproxy docker image (#580)
Co-authored-by: dockerzhang <do...@tencent.com>
---
.../bin/{start.sh => dataproxy-start.sh} | 3 +-
.../bin/{stop.sh => dataproxy-stop.sh} | 0
inlong-dataproxy/bin/prepare_env.sh | 2 +-
.../{bin/start.sh => conf/common.properties} | 8 +-
inlong-dataproxy/dataproxy-dist/pom.xml | 2 +-
.../src/main/assembly/{bin.xml => assembly.xml} | 1 +
.../prepare_env.sh => dataproxy-docker/Dockerfile} | 21 +++--
inlong-dataproxy/dataproxy-docker/README.md | 15 +++
.../dataproxy-docker/dataproxy-docker.sh | 30 ++++++
inlong-dataproxy/dataproxy-docker/pom.xml | 101 +++++++++++++++++++++
.../inlong/dataproxy/config/ConfigManager.java | 2 +-
inlong-dataproxy/pom.xml | 1 +
12 files changed, 169 insertions(+), 17 deletions(-)
diff --git a/inlong-dataproxy/bin/start.sh b/inlong-dataproxy/bin/dataproxy-start.sh
similarity index 92%
copy from inlong-dataproxy/bin/start.sh
copy to inlong-dataproxy/bin/dataproxy-start.sh
index 8ef6819..34a8fa1 100755
--- a/inlong-dataproxy/bin/start.sh
+++ b/inlong-dataproxy/bin/dataproxy-start.sh
@@ -18,5 +18,6 @@
# specific language governing permissions and limitations
# under the License.
#
+cd "$(dirname "$0")"/../
chmod 777 bin/flume-ng
-nohup bin/flume-ng agent --conf conf/ -f conf/flume.conf -n agent1 --no-reload-conf > nohup.out 2>&1 &
\ No newline at end of file
+nohup bin/flume-ng agent --conf conf/ -f conf/flume.conf -n agent1 --no-reload-conf > dataproxy.log 2>&1 &
\ No newline at end of file
diff --git a/inlong-dataproxy/bin/stop.sh b/inlong-dataproxy/bin/dataproxy-stop.sh
similarity index 100%
rename from inlong-dataproxy/bin/stop.sh
rename to inlong-dataproxy/bin/dataproxy-stop.sh
diff --git a/inlong-dataproxy/bin/prepare_env.sh b/inlong-dataproxy/bin/prepare_env.sh
index e0d4cdb..5cc59f4 100755
--- a/inlong-dataproxy/bin/prepare_env.sh
+++ b/inlong-dataproxy/bin/prepare_env.sh
@@ -19,7 +19,7 @@
# under the License.
#
-cd ../conf
+cd "$(dirname "$0")"/../conf
for i in {mx.properties,transfer.properties,weight.properties,common.properties,blacklist.properties,bid_mapping.properties,dc_mapping.properties,topics.properties,tube_switch.properties}
do
diff --git a/inlong-dataproxy/bin/start.sh b/inlong-dataproxy/conf/common.properties
old mode 100755
new mode 100644
similarity index 84%
rename from inlong-dataproxy/bin/start.sh
rename to inlong-dataproxy/conf/common.properties
index 8ef6819..33d1f4b
--- a/inlong-dataproxy/bin/start.sh
+++ b/inlong-dataproxy/conf/common.properties
@@ -1,5 +1,3 @@
-#!/bin/bash
-#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,5 +16,7 @@
# specific language governing permissions and limitations
# under the License.
#
-chmod 777 bin/flume-ng
-nohup bin/flume-ng agent --conf conf/ -f conf/flume.conf -n agent1 --no-reload-conf > nohup.out 2>&1 &
\ No newline at end of file
+# cluter id is for future use please write 1
+cluster_id=1
+# manager open api address
+manager_hosts=ip:port
diff --git a/inlong-dataproxy/dataproxy-dist/pom.xml b/inlong-dataproxy/dataproxy-dist/pom.xml
index 0a74178..b8247e0 100644
--- a/inlong-dataproxy/dataproxy-dist/pom.xml
+++ b/inlong-dataproxy/dataproxy-dist/pom.xml
@@ -59,7 +59,7 @@
<configuration>
<finalName>apache-inlong-dataproxy-${project.version}</finalName>
<descriptors>
- <descriptor>src/main/assembly/bin.xml</descriptor>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
</descriptors>
<tarLongFileMode>gnu</tarLongFileMode>
</configuration>
diff --git a/inlong-dataproxy/dataproxy-dist/src/main/assembly/bin.xml b/inlong-dataproxy/dataproxy-dist/src/main/assembly/assembly.xml
similarity index 99%
rename from inlong-dataproxy/dataproxy-dist/src/main/assembly/bin.xml
rename to inlong-dataproxy/dataproxy-dist/src/main/assembly/assembly.xml
index 2c25a19..7aa9551 100644
--- a/inlong-dataproxy/dataproxy-dist/src/main/assembly/bin.xml
+++ b/inlong-dataproxy/dataproxy-dist/src/main/assembly/assembly.xml
@@ -26,6 +26,7 @@
<formats>
<format>dir</format>
+ <format>tar.gz</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<dependencySets>
diff --git a/inlong-dataproxy/bin/prepare_env.sh b/inlong-dataproxy/dataproxy-docker/Dockerfile
old mode 100755
new mode 100644
similarity index 68%
copy from inlong-dataproxy/bin/prepare_env.sh
copy to inlong-dataproxy/dataproxy-docker/Dockerfile
index e0d4cdb..5369d9d
--- a/inlong-dataproxy/bin/prepare_env.sh
+++ b/inlong-dataproxy/dataproxy-docker/Dockerfile
@@ -1,5 +1,3 @@
-#!/bin/bash
-#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,10 +16,15 @@
# specific language governing permissions and limitations
# under the License.
#
-
-cd ../conf
-
-for i in {mx.properties,transfer.properties,weight.properties,common.properties,blacklist.properties,bid_mapping.properties,dc_mapping.properties,topics.properties,tube_switch.properties}
- do
- touch $i
-done
\ No newline at end of file
+FROM openjdk:8-jdk
+# add tarball from target output
+ARG DATAPROXY_TARBALL
+ADD ${DATAPROXY_TARBALL} /opt/inlong-dataproxy
+EXPOSE 46801
+ENV MANAGER_OPENAPI_IP=127.0.0.1
+ENV MANAGER_OPENAPI_PORT=8082
+ENV TUBMQ_MASTER_LIST=127.0.0.1:8715
+WORKDIR /opt/inlong-dataproxy
+ADD dataproxy-docker.sh bin/
+RUN chmod +x bin/dataproxy-docker.sh
+CMD ["bin/dataproxy-docker.sh"]
diff --git a/inlong-dataproxy/dataproxy-docker/README.md b/inlong-dataproxy/dataproxy-docker/README.md
new file mode 100644
index 0000000..30aed04
--- /dev/null
+++ b/inlong-dataproxy/dataproxy-docker/README.md
@@ -0,0 +1,15 @@
+#### InLong DataProxy docker image
+InLong DataProxy is available for development and experience.
+
+##### Pull Image
+```
+docker pull inlong/dataproxy:latest
+```
+
+##### Start Container
+```
+docker run -d --name dataproxy \
+-e MANAGER_OPENAPI_IP=manager_openapi_ip \
+-e MANAGER_OPENAPI_PORT=manager_openapi_port \
+-e TUBMQ_MASTER_LIST=tube_master_address inlong/dataproxy
+```
\ No newline at end of file
diff --git a/inlong-dataproxy/dataproxy-docker/dataproxy-docker.sh b/inlong-dataproxy/dataproxy-docker/dataproxy-docker.sh
new file mode 100644
index 0000000..e92beaf
--- /dev/null
+++ b/inlong-dataproxy/dataproxy-docker/dataproxy-docker.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+file_path=$(cd "$(dirname "$0")"/../;pwd)
+# config
+sed -i "s/TUBE_LIST/$TUBMQ_MASTER_LIST/g" ${file_path}/conf/flume.conf
+cat <<EOF > ${file_path}/conf/common.properties
+manager_hosts=$MANAGER_OPENAPI_IP:$MANAGER_OPENAPI_PORT
+EOF
+# start
+sh ${file_path}/bin/prepare_env.sh
+sh ${file_path}/bin/dataproxy-start.sh
+sleep 3
+# keep alive
+tail -F ${file_path}/logs/flume.log
diff --git a/inlong-dataproxy/dataproxy-docker/pom.xml b/inlong-dataproxy/dataproxy-docker/pom.xml
new file mode 100644
index 0000000..3f735f5
--- /dev/null
+++ b/inlong-dataproxy/dataproxy-docker/pom.xml
@@ -0,0 +1,101 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>inlong-dataproxy</artifactId>
+ <version>0.9.0-incubating</version>
+ </parent>
+ <artifactId>dataproxy-docker</artifactId>
+ <modelVersion>4.0.0</modelVersion>
+ <name>Apache InLong - DataProxy Docker</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.inlong</groupId>
+ <artifactId>dataproxy-dist</artifactId>
+ <version>${project.parent.version}</version>
+ <classifier>bin</classifier>
+ <type>tar.gz</type>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ <profiles>
+ <profile>
+ <id>docker</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.spotify</groupId>
+ <artifactId>dockerfile-maven-plugin</artifactId>
+ <version>${dockerfile-maven.version}</version>
+ <executions>
+ <execution>
+ <id>default</id>
+ <goals>
+ <goal>build</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>tag-and-push-latest</id>
+ <goals>
+ <goal>tag</goal>
+ <goal>push</goal>
+ </goals>
+ <configuration>
+ <repository>${docker.organization}/dataproxy</repository>
+ <tag>latest</tag>
+ </configuration>
+ </execution>
+ </executions>
+ <configuration>
+ <repository>${docker.organization}/dataproxy</repository>
+ <pullNewerImage>false</pullNewerImage>
+ <tag>${project.version}</tag>
+ <buildArgs>
+ <DATAPROXY_TARBALL>target/dataproxy-dist-${project.version}-bin.tar.gz</DATAPROXY_TARBALL>
+ </buildArgs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-tarball</id>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <phase>generate-resources</phase>
+ <configuration>
+ <outputDirectory>${project.build.directory}/</outputDirectory>
+ <includeArtifactIds>dataproxy-dist</includeArtifactIds>
+ <excludeTransitive>true</excludeTransitive>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --git a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/ConfigManager.java b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/ConfigManager.java
index e04bfbc..0bb62c8 100644
--- a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/ConfigManager.java
+++ b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/ConfigManager.java
@@ -53,7 +53,7 @@ public class ConfigManager {
private static ConfigManager instance = null;
private final PropertiesConfigHolder commonConfig =
- new PropertiesConfigHolder("common.properties");
+ new PropertiesConfigHolder("conf/common.properties");
private final PropertiesConfigHolder topicConfig =
new PropertiesConfigHolder("topics.properties");
private final MxPropertiesHolder mxConfig = new MxPropertiesHolder("mx.properties");
diff --git a/inlong-dataproxy/pom.xml b/inlong-dataproxy/pom.xml
index 168d3fa..777e2a6 100644
--- a/inlong-dataproxy/pom.xml
+++ b/inlong-dataproxy/pom.xml
@@ -32,6 +32,7 @@
<modules>
<module>dataproxy-source</module>
<module>dataproxy-dist</module>
+ <module>dataproxy-docker</module>
</modules>
<properties>