You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by he...@apache.org on 2022/04/05 08:18:07 UTC
[incubator-inlong] branch master updated: [INLONG-3528][InLong] Unify the log directory for manager and audit (#3529)
This is an automated email from the ASF dual-hosted git repository.
healchow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 80fc3971b [INLONG-3528][InLong] Unify the log directory for manager and audit (#3529)
80fc3971b is described below
commit 80fc3971b9eb124939ec5f204cc5866742414840
Author: dockerzhang <do...@tencent.com>
AuthorDate: Tue Apr 5 16:18:01 2022 +0800
[INLONG-3528][InLong] Unify the log directory for manager and audit (#3529)
---
bin/inlong-daemon | 2 +-
conf/inlong.conf | 2 +-
inlong-agent/agent-docker/README.md | 40 +---------------------
inlong-audit/audit-docker/audit-docker.sh | 2 +-
inlong-audit/bin/proxy-start.sh | 15 +++++---
inlong-audit/bin/store-start.sh | 16 +++++----
inlong-audit/conf/logback.xml | 3 +-
inlong-manager/manager-docker/README.md | 13 +------
inlong-manager/manager-docker/manager-docker.sh | 4 +--
.../main/resources/flink-sort-plugin.properties | 2 +-
.../service/core/plugin/PluginClassLoader.java | 3 +-
inlong-manager/manager-web/bin/startup.sh | 29 +++-------------
.../src/main/resources/logback-spring.xml | 2 +-
inlong-sort/sort-single-tenant/pom.xml | 1 -
14 files changed, 35 insertions(+), 99 deletions(-)
diff --git a/bin/inlong-daemon b/bin/inlong-daemon
index 492202430..b1ab18d21 100644
--- a/bin/inlong-daemon
+++ b/bin/inlong-daemon
@@ -136,7 +136,7 @@ start_inlong_agent() {
# register dataproxy for standalone
register_service() {
- curl --header "Content-Type: application/json" --request POST http://localhost:8083/api/inlong/manager/openapi/cluster/save --data '
+ curl --header "Content-Type: application/json" --request POST http://${manager_server_hostname}:${manager_server_port}/api/inlong/manager/openapi/cluster/save --data '
{
"name": "default_dataproxy",
"type": "DATA_PROXY",
diff --git a/conf/inlong.conf b/conf/inlong.conf
index e4626b236..7698ba823 100644
--- a/conf/inlong.conf
+++ b/conf/inlong.conf
@@ -40,7 +40,7 @@ dashboard_docker_image=inlong/dashboard:latest
dashboard_mapping_port=80
############## Manager Configuration ##############
-# The default value is dev
+# The default value is dev, prod is available
spring_profiles_active=dev
# manager service IP
manager_server_hostname=127.0.0.1
diff --git a/inlong-agent/agent-docker/README.md b/inlong-agent/agent-docker/README.md
index 343bedab1..03fc44d3f 100644
--- a/inlong-agent/agent-docker/README.md
+++ b/inlong-agent/agent-docker/README.md
@@ -11,42 +11,4 @@ docker pull inlong/agent:latest
docker run -d --name agent -p 8008:8008 \
-e MANAGER_OPENAPI_IP=manager_opeapi_ip -e DATAPROXY_IP=dataproxy_ip \
-e MANAGER_OPENAPI_PORT=8082 -e DATAPROXY_PORT=46801 inlong/agent
-```
-#### Add A Job
-```
-curl --location --request POST 'http://localhost:8008/config/job' \
---header 'Content-Type: application/json' \
---data '{
-"job": {
-"dir": {
-"path": "",
-"pattern": "/data/inlong-agent/test.log"
-},
-"trigger": "org.apache.inlong.agent.plugin.trigger.DirectoryTrigger",
-"id": 1,
-"thread": {
-"running": {
-"core": "4"
-}
-},
-"name": "fileAgentTest",
-"source": "org.apache.inlong.agent.plugin.sources.TextFileSource",
-"sink": "org.apache.inlong.agent.plugin.sinks.ProxySink",
-"channel": "org.apache.inlong.agent.plugin.channel.MemoryChannel"
-},
-"proxy": {
-"inlongGroupId": "group10",
-"inlongStreamId": "group10"
-},
-"op": "add"
-}'
-```
-
-The meaning of each parameter is ：
-
-- job.dir.pattern: Configure the read file path, which can include regular expressions
-- job.trigger: Trigger name, the default is DirectoryTrigger, the function is to monitor the files under the folder to generate events
-- job.source: The type of data source used, the default is TextFileSource, which reads text files
-- job.sink：The type of writer used, the default is ProxySink, which sends messages to the proxy
-- proxy.inlongGroupId: The inlongGroupId used when writing proxy
-- proxy.inlongStreamId: The inlongStreamId used when writing proxy
\ No newline at end of file
+```
\ No newline at end of file
diff --git a/inlong-audit/audit-docker/audit-docker.sh b/inlong-audit/audit-docker/audit-docker.sh
index 756acd25d..2b0f5a27c 100644
--- a/inlong-audit/audit-docker/audit-docker.sh
+++ b/inlong-audit/audit-docker/audit-docker.sh
@@ -44,4 +44,4 @@ bash +x ${file_path}/bin/proxy-start.sh
bash +x ${file_path}/bin/store-start.sh
sleep 3
# keep alive
-tail -F ${file_path}/logs/audit-*
+tail -F ${file_path}/logs/audit.log
diff --git a/inlong-audit/bin/proxy-start.sh b/inlong-audit/bin/proxy-start.sh
index 80a72ae8a..b3bc3d0a7 100644
--- a/inlong-audit/bin/proxy-start.sh
+++ b/inlong-audit/bin/proxy-start.sh
@@ -18,7 +18,11 @@
# specific language governing permissions and limitations
# under the License.
#
-cd "$(dirname "$0")"/../
+base_dir=$(
+ cd $(dirname $0)
+ cd ..
+ pwd
+)
error() {
local msg=$1
@@ -41,13 +45,16 @@ if [ -n "$1" ]; then
MQ_TYPE=$1
fi
-basedir="$(pwd)"
CONFIG_FILE="audit-proxy-${MQ_TYPE}.conf"
CONFIG_FILE_WITH_COFING_PATH="conf/${CONFIG_FILE}"
-CONFIG_FILE_WITH_PATH="${basedir}/conf/${CONFIG_FILE}"
+CONFIG_FILE_WITH_PATH="${base_dir}/conf/${CONFIG_FILE}"
+LOG_DIR="${base_dir}/logs"
+if [ ! -d "${LOG_DIR}" ]; then
+ mkdir ${LOG_DIR}
+fi
if [ -f "$CONFIG_FILE_WITH_PATH" ]; then
- nohup bin/audit-proxy agent --conf conf/ -f "${CONFIG_FILE_WITH_COFING_PATH}" -n agent1 --no-reload-conf >audit-proxy.log 2>&1 &
+ nohup bin/audit-proxy agent --conf conf/ -f "${CONFIG_FILE_WITH_COFING_PATH}" -n agent1 --no-reload-conf 1>${LOG_DIR}/proxy.log 2>${LOG_DIR}/proxy-error.log &
else
error "${CONFIG_FILE_WITH_PATH} is not exist! start failed!" 1
fi
diff --git a/inlong-audit/bin/store-start.sh b/inlong-audit/bin/store-start.sh
index b46bd56e9..f24a4e0ed 100644
--- a/inlong-audit/bin/store-start.sh
+++ b/inlong-audit/bin/store-start.sh
@@ -18,11 +18,14 @@
# specific language governing permissions and limitations
# under the License.
#
-bin_dir=$(dirname $0)
-base_dir=`cd -P $bin_dir/..;pwd`
-cd ..
+base_dir=$(
+ cd $(dirname $0)
+ cd ..
+ pwd
+)
PID=$(ps -ef | grep "audit-store" | grep -v grep | awk '{ print $2}')
+LOG_DIR="${basedir}/logs"
if [ -n "$PID" ]; then
echo "Application has already started."
@@ -39,8 +42,8 @@ else
JAVA=$JAVA_HOME/bin/java
fi
-if [ ! -d "${base_dir}/logs" ]; then
- mkdir ${base_dir}/logs
+if [ ! -d "${LOG_DIR}" ]; then
+ mkdir ${LOG_DIR}
fi
JAVA_OPTS="-server -XX:SurvivorRatio=2 -XX:+UseParallelGC"
@@ -54,8 +57,7 @@ JAVA_OPTS="${JAVA_OPTS} ${HEAP_OPTS}"
SERVERJAR=`ls -lt ${base_dir}/lib |grep audit-store | head -2 | tail -1 | awk '{print $NF}'`
-nohup $JAVA $JAVA_OPTS -Dloader.path="$base_dir/conf,$base_dir/lib/" -jar "$base_dir/lib/$SERVERJAR" > $base_dir/logs/audit-store.log 2>&1 < /dev/null &
-
+nohup $JAVA $JAVA_OPTS -Dloader.path="$base_dir/conf,$base_dir/lib/" -jar "$base_dir/lib/$SERVERJAR" 1>${LOG_DIR}/store.log 2>${LOG_DIR}/store-error.log &
PIDFILE="$base_dir/bin/PID"
PID=$(ps -ef | grep "$base_dir" | grep -v grep | awk '{ print $2}')
diff --git a/inlong-audit/conf/logback.xml b/inlong-audit/conf/logback.xml
index 92f0b8177..1d74464f8 100644
--- a/inlong-audit/conf/logback.xml
+++ b/inlong-audit/conf/logback.xml
@@ -18,11 +18,10 @@
under the License.
-->
<configuration scan="true" scanPeriod="60s">
- <property name="PROJECT_NAME" value="audit"/>
<property name="LOG_BACK_DIR" value="logs"/>
<appender name="INFO_ROLLING" class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${LOG_BACK_DIR}/audit-proxy.log</file>
+ <file>${LOG_BACK_DIR}/audit.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!-- rollover daily -->
<fileNamePattern>${LOG_BACK_DIR}/info-%d{yyyy-MM-dd}.%i.gz</fileNamePattern>
diff --git a/inlong-manager/manager-docker/README.md b/inlong-manager/manager-docker/README.md
index 184900081..af3c04201 100644
--- a/inlong-manager/manager-docker/README.md
+++ b/inlong-manager/manager-docker/README.md
@@ -16,16 +16,5 @@ docker run -d --name manager -p 8083:8083 \
-e JDBC_URL=127.0.0.1:3306 \
-e USERNAME=root \
-e PASSWORD=inlong \
--e TUBE_MANAGER=127.0.0.1:8089 \
--e TUBE_MASTER=127.0.0.1:8715 \
--e ZK_URL=127.0.0.1:2181 \
--e ZK_ROOT=inlong_hive \
--e SORT_APP_NAME=inlong_hive \
inlong/manager:latest
-```
-
-### Add A Job
-
-Check if the service started successfully:
-
-[http://localhost:8083/api/inlong/manager/doc.html](http://localhost:8083/api/inlong/manager/doc.html)
+```
\ No newline at end of file
diff --git a/inlong-manager/manager-docker/manager-docker.sh b/inlong-manager/manager-docker/manager-docker.sh
index 9b10389a3..b32c06254 100644
--- a/inlong-manager/manager-docker/manager-docker.sh
+++ b/inlong-manager/manager-docker/manager-docker.sh
@@ -33,8 +33,6 @@ sed -i "s/127.0.0.1:3306/${JDBC_URL}/g" "${conf_file}"
sed -i "s/datasource.druid.username=.*$/datasource.druid.username=${USERNAME}/g" "${conf_file}"
sed -i "s/datasource.druid.password=.*$/datasource.druid.password=${PASSWORD}/g" "${conf_file}"
-sed -i "s/cluster.zk.url=.*$/cluster.zk.url=${ZK_URL}/g" "${conf_file}"
-
# startup the application
JAVA_OPTS="-Dspring.profiles.active=${ACTIVE_PROFILE}"
@@ -51,4 +49,4 @@ fi
sh "${file_path}"/bin/startup.sh "${JAVA_OPTS}"
sleep 3
# keep alive
-tail -F "${file_path}"/log/manager-web.log
+tail -F "${file_path}"/logs/manager-web.log
diff --git a/inlong-manager/manager-plugins/src/main/resources/flink-sort-plugin.properties b/inlong-manager/manager-plugins/src/main/resources/flink-sort-plugin.properties
index d67b92f88..7a9bd0cf1 100644
--- a/inlong-manager/manager-plugins/src/main/resources/flink-sort-plugin.properties
+++ b/inlong-manager/manager-plugins/src/main/resources/flink-sort-plugin.properties
@@ -25,7 +25,7 @@
# Flink host split by coma if more than one host, such as 'host1,host2'
flink.rest.address=127.0.0.1
# Flink port
-flink.rest.port=8080
+flink.rest.port=8081
# Flink jobmanager port
flink.jobmanager.port=6123
# Flink savepoint directory
diff --git a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/plugin/PluginClassLoader.java b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/plugin/PluginClassLoader.java
index e145828d8..581275de3 100644
--- a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/plugin/PluginClassLoader.java
+++ b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/plugin/PluginClassLoader.java
@@ -119,9 +119,10 @@ public class PluginClassLoader extends URLClassLoader {
List<PluginDefinition> definitions = new ArrayList();
for (File jarFile : pluginDirectory.listFiles()) {
if (!jarFile.getName().endsWith(".jar")) {
- log.warn("{}' is not plugin jar , please check", jarFile);
+ log.warn("{} is not valid plugin jar, skip to load", jarFile);
continue;
}
+ log.info("{} is valid plugin jar, start to load", jarFile);
JarFile pluginJar = new JarFile(jarFile);
String pluginDef = readPluginDef(pluginJar);
pluginDef = pluginDef.replaceAll("[\\x00]+", "");
diff --git a/inlong-manager/manager-web/bin/startup.sh b/inlong-manager/manager-web/bin/startup.sh
index 3e54b0028..be85bd90a 100755
--- a/inlong-manager/manager-web/bin/startup.sh
+++ b/inlong-manager/manager-web/bin/startup.sh
@@ -20,10 +20,8 @@
#======================================================================
# Project start shell script
-# config directory: configuration file directory
-# log directory: project operation log directory
-# log/startup.log: record startup log
-# log/back directory: Project running log backup directory
+# logs directory: project operation log directory
+# logs/back directory: Project running log backup directory
# nohup background process
#
#======================================================================
@@ -59,13 +57,7 @@ CLASSPATH=${CONFIG_DIR}:${JAR_LIBS}:${JAR_MAIN}
MAIN_CLASS=org.apache.inlong.manager.web.InLongWebApplication
# Project log output absolute path
-LOG_DIR=${BASE_PATH}"/log"
-LOG_FILE="${LOG_DIR}/sout-manager-web.log"
-# Log backup directory
-LOG_BACK_DIR="${LOG_DIR}/back/"
-
-# Project startup log output absolute path
-LOG_STARTUP_PATH="${LOG_DIR}/startup.log"
+LOG_DIR=${BASE_PATH}"/logs"
# current time
NOW=$(date +'%Y-%m-%m-%H-%M-%S')
@@ -76,19 +68,6 @@ if [ ! -d "${LOG_DIR}" ]; then
mkdir "${LOG_DIR}"
fi
-# If the log/back folder does not exist, create a folder
-if [ ! -d "${LOG_BACK_DIR}" ]; then
- mkdir "${LOG_BACK_DIR}"
-fi
-
-# If the project log exists, rename the backup
-if [ -f "${LOG_FILE}" ]; then
- mv ${LOG_FILE} "${LOG_BACK_DIR}/${APPLICATION}_back_${NOW}.log"
-fi
-
-# Create a new project run log
-echo "" >${LOG_FILE}
-
# JVM Configuration
JAVA_OPT="-server -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m -XX:-OmitStackTraceInFastThrow"
@@ -112,4 +91,4 @@ JAVA_OPT="${JAVA_OPT} -XX:+IgnoreUnrecognizedVMOptions -XX:+UseConcMarkSweepGC -
#JAVA_OPT="${JAVA_OPT} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8081"
# Execute the startup command: start the project in the background, and output the log to the logs folder under the project root directory
-nohup java ${JAVA_OPT} -Dlog4j2.formatMsgNoLookups=true -Dlog4j.formatMsgNoLookups=true -cp ${CLASSPATH} ${MAIN_CLASS} 1>${LOG_FILE} 2>${LOG_DIR}/error.log &
\ No newline at end of file
+nohup java ${JAVA_OPT} -Dlog4j2.formatMsgNoLookups=true -Dlog4j.formatMsgNoLookups=true -cp ${CLASSPATH} ${MAIN_CLASS} 1>/dev/null 2>${LOG_DIR}/error.log &
\ No newline at end of file
diff --git a/inlong-manager/manager-web/src/main/resources/logback-spring.xml b/inlong-manager/manager-web/src/main/resources/logback-spring.xml
index 5670bc85e..2fd5e2b48 100644
--- a/inlong-manager/manager-web/src/main/resources/logback-spring.xml
+++ b/inlong-manager/manager-web/src/main/resources/logback-spring.xml
@@ -22,7 +22,7 @@
<contextName>logback</contextName>
<!-- The defined value will be inserted into the logger context. After definition, you can use "${}" to use variables. -->
- <property name="log.path" value="${LOG_PATH}"/>
+ <property name="log.path" value="logs"/>
<!--0. Log format and color rendering -->
<!-- The rendering class that the color log depends on -->
diff --git a/inlong-sort/sort-single-tenant/pom.xml b/inlong-sort/sort-single-tenant/pom.xml
index 1b892dd3c..ab2046674 100644
--- a/inlong-sort/sort-single-tenant/pom.xml
+++ b/inlong-sort/sort-single-tenant/pom.xml
@@ -118,7 +118,6 @@
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-runtime-blink_${flink.scala.binary.version}</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>