You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2022/03/31 03:32:23 UTC
[incubator-inlong] branch master updated: [INLONG-3481][InLong] Optimize the standalone script (#3482)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 059948f [INLONG-3481][InLong] Optimize the standalone script (#3482)
059948f is described below
commit 059948f655c291d7ba374e8bee708e71cc1409df
Author: dockerzhang <do...@apache.org>
AuthorDate: Thu Mar 31 11:32:17 2022 +0800
[INLONG-3481][InLong] Optimize the standalone script (#3482)
---
bin/init-config.sh | 104 ++------
bin/inlong-daemon | 279 ++++++---------------
conf/inlong.conf | 140 +++--------
docker/docker-compose/sql/tubemanager.sql | 24 --
inlong-manager/manager-web/bin/startup.sh | 38 +--
.../tubemq-docker/tubemq-manager/manager-docker.sh | 2 +-
.../tubemq-manager/sql/apache_tube_manager.sql | 2 +-
.../src/main/resources/application.properties | 2 +-
8 files changed, 140 insertions(+), 451 deletions(-)
diff --git a/bin/init-config.sh b/bin/init-config.sh
index 2635293..075ba00 100644
--- a/bin/init-config.sh
+++ b/bin/init-config.sh
@@ -21,87 +21,40 @@ INLONG_HOME=$(
cd ..
pwd
)
-echo $INLONG_HOME
source $INLONG_HOME/conf/inlong.conf
init_inlong_agent() {
echo "Init agent configuration parameters"
cd $INLONG_HOME/inlong-agent/conf
- sed -i 's/agent.local.ip=.*/'''agent.local.ip=${agent_local_ip}'''/g' agent.properties
+ sed -i 's/agent.local.ip=.*/'''agent.local.ip=${local_ip}'''/g' agent.properties
+ sed -i 's/agent.http.port=.*/'''agent.http.port=${agent_port}'''/g' agent.properties
sed -i 's/agent.manager.vip.http.host=.*/'''agent.manager.vip.http.host=${manager_server_hostname}'''/g' agent.properties
sed -i 's/agent.manager.vip.http.port=.*/'''agent.manager.vip.http.port=${manager_server_port}'''/g' agent.properties
- sed -i 's/audit.proxys=.*/'''audit.proxys=${audit_proxys_ip}:10081'''/g' agent.properties
- sed -i 's/agent.prometheus.enable=.*/'''agent.prometheus.enable=${agent_prometheus_enable}'''/g' agent.properties
- sed -i 's/agent.audit.enable=.*/'''agent.audit.enable=${agent_audit_enable}'''/g' agent.properties
+ sed -i 's/audit.proxys=.*/'''audit.proxys=${audit_proxys_ip}:${audit_proxys_port}'''/g' agent.properties
}
init_inlong_audit() {
- if [ $source_type == "pulsar" ]; then
- echo "Init audit configuration parameters"
- cd $INLONG_HOME/inlong-audit/conf
+ echo "Init audit configuration parameters"
+ cd $INLONG_HOME/inlong-audit/conf
+ sed -i 's#jdbc:mysql://.*apache_inlong_audit#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/apache_inlong_audit'''#g' application.properties
+ sed -i 's/spring.datasource.druid.username=.*/'''spring.datasource.druid.username=${spring_datasource_username}'''/g' application.properties
+ sed -i 's/spring.datasource.druid.password=.*/'''spring.datasource.druid.password=${spring_datasource_password}'''/g' application.properties
+ if [ $mq_type == "pulsar" ]; then
sed -i 's#pulsar://.*#'''${pulsar_service_url}'''#g' audit-proxy-pulsar.conf
sed -i 's#pulsar://.*#'''${pulsar_service_url}'''#g' application.properties
- sed -i 's#jdbc:mysql://.*apache_inlong_audit#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/apache_inlong_audit'''#g' application.properties
- sed -i 's/spring.datasource.druid.username=.*/'''spring.datasource.druid.username=${spring_datasource_username}'''/g' application.properties
- sed -i 's/spring.datasource.druid.password=.*/'''spring.datasource.druid.password=${spring_datasource_password}'''/g' application.properties
+ fi
+ if [ $mq_type == "tubemq" ]; then
+ sed -i 's/agent1.sinks.tube-sink-msg1.master-host-port-list=.*/'''agent1.sinks.tube-sink-msg1.master-host-port-list=${tube_master_url}'''/g' audit-proxy-tube.conf
+ sed -i 's/agent1.sinks.tube-sink-msg2.master-host-port-list=.*/'''agent1.sinks.tube-sink-msg2.master-host-port-list=${tube_master_url}'''/g' audit-proxy-tube.conf
+ sed -i 's/audit.tube.masterlist=.*/'''audit.tube.masterlist=${tube_master_url}'''/g' application.properties
fi
}
init_inlong_dataproxy() {
echo "Init dataproxy configuration parameters"
cd $INLONG_HOME/inlong-dataproxy/conf
- if [ $source_type == "pulsar" ]; then
- if [ -f "flume-mulit-pulsar-tcp-example.conf" ]; then
- mv flume.conf flume-tubemq.conf
- mv flume-mulit-pulsar-tcp-example.conf flume.conf
- fi
- sed -i 's#pulsar://.*#'''${pulsar_service_url}'''#g' flume.conf
- else
- if [ -f "flume-tubemq.conf" ]; then
- mv flume.conf flume-mulit-pulsar-tcp-example.conf
- mv flume-tubemq.conf flume.conf
- fi
- sed -i 's/master-host-port-list.*/'''master-host-port-list=${tubemaster_hostname}:${tubemaster_port}'''/g' flume.conf
- fi
- sed -i 's/manager_hosts=.*/'''manager_hosts=${manager_server_hostname}:${manager_server_port}'''/g' common.properties
- sed -i 's/audit.proxys=.*/'''audit.proxys=${audit_proxys_ip}:10081'''/g' common.properties
-}
-
-init_inlong_tubemaster() {
- echo "Init tubemq server configuration parameters"
- cd $INLONG_HOME/inlong-tubemq-server/conf
- echo "Init tubemq master configuration"
- sed -i 's/hostName=.*/'''hostName=${tubemaster_hostname}'''/g' master.ini
- sed -i 's/port=.*/'''port=${tubemaster_port}'''/g' master.ini
- sed -i 's/webPort=.*/'''webPort=${tubemaster_webport}'''/g' master.ini
- sed -i 's#metaDataPath=.*#'''metaDataPath=${metadata_path}'''#g' master.ini
- sed -i 's/;metaDataPath/metaDataPath/g' master.ini
- sed -i 's/confModAuthToken=.*/'''confModAuthToken=${confmod_authtoken}'''/g' master.ini
- sed -i 's/zkServerAddr=.*/'''zkServerAddr=${zkserver_addr}'''/g' master.ini
- sed -i 's/repHelperHost=.*/'''repHelperHost=${tubemaster_hostname}:9001'''/g' master.ini
- sed -i 's/;repHelperHost/'''repHelperHost'''/g' master.ini
-}
-
-init_inlong_tubebroker() {
- echo "Init tubemq broker configuration"
- cd $INLONG_HOME/inlong-tubemq-server/conf
- sed -i 's/brokerId=.*/'''brokerId=${broker_id}'''/g' broker.ini
- sed -i 's/hostName=.*/'''hostName=${tubebroker_hostname}'''/g' broker.ini
- sed -i 's/port=.*/'''port=${tubebroker_port}'''/g' broker.ini
- sed -i 's/webPort=.*/'''webPort=${tubebroker_webport}'''/g' broker.ini
- sed -i 's/masterAddressList=.*/'''masterAddressList=${tubemaster_hostname}:${tubemaster_port}'''/g' broker.ini
- sed -i 's#primaryPath=.*#'''primaryPath=${primary_path}'''#g' broker.ini
- sed -i 's/zkServerAddr=.*/'''zkServerAddr=${zkserver_addr}'''/g' broker.ini
-}
-
-init_inlong_tubemanager() {
- echo "Init tubemq manager configuration"
- cd $INLONG_HOME/inlong-tubemq-manager/conf
- sed -i 's#jdbc:mysql://.*tubemanager#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/tubemanager'''#g' application.properties
- sed -i 's/spring.datasource.username=.*/'''spring.datasource.username=${spring_datasource_username}'''/g' application.properties
- sed -i 's/spring.datasource.password=.*/'''spring.datasource.password=${spring_datasource_password}'''/g' application.properties
- sed -i 's/#.*spring.datasource/spring.datasource/g' application.properties
- sed -i 's/server.port=.*/'''server.port=${tube_manager_port}'''/g' application.properties
+ sed -i 's/manager.hosts=.*/'''manager.hosts=${manager_server_hostname}:${manager_server_port}'''/g' common.properties
+ sed -i 's/audit.proxys=.*/'''audit.proxys=${audit_proxys_ip}:${audit_proxys_port}'''/g' common.properties
}
init_inlong_manager() {
@@ -109,29 +62,22 @@ init_inlong_manager() {
cd $INLONG_HOME/inlong-manager/conf
sed -i 's/spring.profiles.active=.*/'''spring.profiles.active=${spring_profiles_active}'''/g' application.properties
sed -i 's/server.port=.*/'''server.port=${manager_server_port}'''/g' application.properties
- sed -i 's#jdbc:mysql://.*apache_inlong_manager#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/apache_inlong_manager'''#g' application-${spring_profiles_active}.properties
- sed -i 's/spring.datasource.druid.username=.*/'''spring.datasource.druid.username=${spring_datasource_username}'''/g' application-${spring_profiles_active}.properties
- sed -i 's/spring.datasource.druid.password=.*/'''spring.datasource.druid.password=${spring_datasource_password}'''/g' application-${spring_profiles_active}.properties
- if [ $source_type == "tubemq" ]; then
- sed -i 's#cluster.tube.manager=.*#'''cluster.tube.manager=http://${tube_manager_ip}:${tube_manager_port}'''#g' application-${spring_profiles_active}.properties
- sed -i 's#cluster.tube.master=.*#'''cluster.tube.master=${tubemaster_hostname}:${tubemaster_port}'''#g' application-${spring_profiles_active}.properties
- else
- sed -i 's#pulsar.adminUrl=.*#'''pulsar.adminUrl=${pulsar_admin_url}'''#g' application-${spring_profiles_active}.properties
- sed -i 's#pulsar.serviceUrl=.*#'''pulsar.serviceUrl=${pulsar_service_url}'''#g' application-${spring_profiles_active}.properties
- sed -i 's/pulsar.defaultTenant=.*/'''pulsar.defaultTenant=${pulsar_default_tenant}'''/g' application-${spring_profiles_active}.properties
+ if [ $spring_profiles_active == "dev" ]; then
+ sed -i 's#jdbc:mysql://.*apache_inlong_manager#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/apache_inlong_manager'''#g' application-dev.properties
+ sed -i 's/spring.datasource.druid.username=.*/'''spring.datasource.druid.username=${spring_datasource_username}'''/g' application-dev.properties
+ sed -i 's/spring.datasource.druid.password=.*/'''spring.datasource.druid.password=${spring_datasource_password}'''/g' application-dev.properties
+ fi
+ if [ $spring_profiles_active == "prod" ]; then
+ sed -i 's#jdbc:mysql://.*apache_inlong_manager#'''jdbc:mysql://${spring_datasource_hostname}:${spring_datasource_port}/apache_inlong_manager'''#g' application-prod.properties
+ sed -i 's/spring.datasource.druid.username=.*/'''spring.datasource.druid.username=${spring_datasource_username}'''/g' application-prod.properties
+ sed -i 's/spring.datasource.druid.password=.*/'''spring.datasource.druid.password=${spring_datasource_password}'''/g' application-prod.properties
fi
- sed -i 's/cluster.zk.url=.*/'''cluster.zk.url=${zkserver_addr}'''/g' application-${spring_profiles_active}.properties
- sed -i 's/cluster.zk.root=.*/'''cluster.zk.root=${cluster_zk_root}'''/g' application-${spring_profiles_active}.properties
- sed -i 's/sort.appName=.*/'''sort.appName=${sort_app_name}'''/g' application-${spring_profiles_active}.properties
}
if [ $# -eq 0 ]; then
init_inlong_agent
init_inlong_audit
init_inlong_dataproxy
- init_inlong_tubemaster
- init_inlong_tubebroker
- init_inlong_tubemanager
init_inlong_manager
else
init_inlong_$1
diff --git a/bin/inlong-daemon b/bin/inlong-daemon
index 92d7088..a67b016 100644
--- a/bin/inlong-daemon
+++ b/bin/inlong-daemon
@@ -22,25 +22,31 @@ INLONG_HOME=$(
cd ..
pwd
)
-TUBEMQDBNAME="tubemanager" # tubemq manager database
-INLONGDBNAME="apache_inlong_manager" # inlong manager Metabase name
source $INLONG_HOME/conf/inlong.conf
command_help() {
echo "Usage: ./inlong-daemon (start|stop) <command>
where command is one of:
audit Run a Audit Service
- tubemaster Run a TubeMQ Master Service
- tubebroker Run a TubeMQ Broker Service
- tubemanager Run a TubeMQ Manager Service
manager Run a Manager Service
dashboard Run a Dashboard Service
dataproxy Run a Dataproxy Service
agent Run a Agent Service
- sort Run a Sort Service
standalone Run a Standalone mode(All Service)"
}
+# wait the service to start
+wait_port_to_listen() {
+ service_name=$1
+ service_port=$2
+ result=$(lsof -Pi :${service_port} -sTCP:LISTEN)
+ while [[ -z "$result" ]]; do
+ echo "waiting $service_name to start, sleep 3s ..."
+ sleep 3
+ result=$(lsof -Pi :${service_port} -sTCP:LISTEN)
+ done
+}
+
# if less than two arguments supplied
if [ $# -lt 2 ]; then
command_help
@@ -57,67 +63,6 @@ init_compo() {
fi
}
-start_inlong_tubemaster() {
- init_compo
- echo "start-up tubemq master ..."
- cd $INLONG_HOME/inlong-tubemq-server/bin
- chmod 755 tubemq.sh
- ./tubemq.sh master start
-}
-
-start_inlong_tubebroker() {
- init_compo
- # Judge whether the TubeMqMaster is started in the current system
- tubemq_master_thread=$($JAVA_HOME/bin/jps | grep MasterStartup)
- if [[ ! -n "$tubemq_master_thread" ]]; then
- echo "The system does not start tubemqMaster or starts abnormally. Please start tubemqMaster normally first"
- exit 1
- fi
- # add broker
- curl -d 'type=op_modify&method=admin_add_broker_configure&brokerId='"$broker_id"'&brokerIp='"$tubebroker_hostname"'&createUser=docker&brokerPort='"$tubebroker_port"'&deletePolicy=delete,168h&numPartitions=3&unflushThreshold=1000&acceptPublish=true&acceptSubscribe=true&unflushInterval=10000&confModAuthToken='"$confmod_authtoken"'' http://$tubemaster_hostname:$tubemaster_webport/webapi.htm
- # online
- curl -d 'type=op_modify&method=admin_online_broker_configure&brokerId='"$broker_id"'&modifyUser=docker&confModAuthToken='"$confmod_authtoken"'' http://$tubemaster_hostname:$tubemaster_webport/webapi.htm
- # broker start
- echo "start-up tubemq broker ..."
- cd $INLONG_HOME/inlong-tubemq-server/bin
- chmod 755 tubemq.sh
- ./tubemq.sh broker start
-}
-
-start_inlong_tubemanager() {
- init_compo
- # Judge whether the TubeMqServer is started in the current system
- tubemq_broker_thread=$($JAVA_HOME/bin/jps | grep BrokerStartup)
- tubemq_master_thread=$($JAVA_HOME/bin/jps | grep MasterStartup)
- if [[ ! -n "$tubemq_broker_thread" && ! -n "$tubemq_master_thread" ]]; then
- echo "The system does not start tubemqServer or starts abnormally. Please start tubemqServer(master and broker) normally first"
- exit 1
- fi
- echo "start-up tubemq manager ..."
- cd $INLONG_HOME/inlong-tubemq-manager/bin
- # create tubemanager database
- create_db_sql="create database IF NOT EXISTS ${TUBEMQDBNAME}"
- mysql -h${spring_datasource_hostname} -P${spring_datasource_port} -u${spring_datasource_username} -p${spring_datasource_password} -e "${create_db_sql}"
- ./start-manager.sh
- sleep 10
- # first init
- flag=$(cat init-tube-cluster.sh | grep "flag=1")
- if [ ! -n "$flag" ]; then
- echo "init shell config"
- sed -i 's/TUBE_MANAGER_IP=.*/'''TUBE_MANAGER_IP=${tube_manager_ip}'''/g' init-tube-cluster.sh
- sed -i 's/TUBE_MANAGER_PORT=.*/'''TUBE_MANAGER_PORT=${tube_manager_port}'''/g' init-tube-cluster.sh
- sed -i 's/TUBE_MASTER_IP=.*/'''TUBE_MASTER_IP=${tubemaster_hostname}'''/g' init-tube-cluster.sh
- sed -i 's/TUBE_MASTER_PORT=.*/'''TUBE_MASTER_PORT=${tubemaster_port}'''/g' init-tube-cluster.sh
- sed -i 's/TUBE_MASTER_WEB_PORT=.*/'''TUBE_MASTER_WEB_PORT=${tubemaster_webport}'''/g' init-tube-cluster.sh
- sed -i 's/TUBE_MASTER_TOKEN=.*/'''TUBE_MASTER_TOKEN=${confmod_authtoken}'''/g' init-tube-cluster.sh
- echo "init tubemq cluster"
- ./init-tube-cluster.sh
- echo -e "\nflag=1" >>init-tube-cluster.sh
- else
- echo "tubemq cluster initialized,skip"
- fi
-}
-
start_inlong_audit() {
init_compo
echo "init apache_inlong_audit"
@@ -127,138 +72,103 @@ start_inlong_audit() {
inlong_num=$(echo $inlong_audit_count | tr -cd "[0-9]")
if [ $inlong_num -eq 0 ]; then
mysql -h${spring_datasource_hostname} -P${spring_datasource_port} -u${spring_datasource_username} -p${spring_datasource_password} <sql/apache_inlong_audit.sql
- else
- echo "apache_inlong_audit database initialized,skip"
fi
- if [ $source_type == "pulsar" ]; then
- cd $INLONG_HOME/inlong-audit/bin
- echo "start-up audit"
- ./proxy-start.sh
- echo "start audit_store"
- ./store-start.sh
+ echo "start audit proxy"
+ if [ $mq_type == "pulsar" ]; then
+ ./bin/proxy-start.sh pulsar
+ fi
+ if [ $mq_type == "tubemq" ]; then
+ ./bin/proxy-start.sh tube
fi
+ echo "start audit store"
+ ./bin/store-start.sh
+ # wait to start
+ wait_port_to_listen audit ${audit_proxys_port}
}
start_inlong_manager() {
init_compo
- echo "start-up inlong manager ..."
+ echo "start manager"
cd $INLONG_HOME/inlong-manager
# Whether the database table exists. If it does not exist, initialize the database and skip if it exists.
- select_db_sql="SELECT COUNT(*) FROM information_schema.TABLES WHERE table_schema = '${INLONGDBNAME}'"
+ select_db_sql="SELECT COUNT(*) FROM information_schema.TABLES WHERE table_schema = 'apache_inlong_manager'"
inlong_manager_count=$(mysql -h${spring_datasource_hostname} -P${spring_datasource_port} -u${spring_datasource_username} -p${spring_datasource_password} -e "${select_db_sql}")
inlong_num=$(echo $inlong_manager_count | tr -cd "[0-9]")
if [ $inlong_num -eq 0 ]; then
echo "init apache_inlong_manager database"
mysql -h${spring_datasource_hostname} -P${spring_datasource_port} -u${spring_datasource_username} -p${spring_datasource_password} <sql/apache_inlong_manager.sql
- else
- echo "apache_inlong_manager database initialized,skip"
fi
- cd $INLONG_HOME/inlong-manager/bin
- ./startup.sh
+ ./bin/startup.sh
+ # wait to start
+ wait_port_to_listen manager ${manager_server_port}
}
start_inlong_dashboard() {
- echo "start-up inlong dashboard ..."
+ echo "start dashboard"
if [[ "$manager_server_hostname" == "localhost" || "$manager_server_hostname" == "127.0.0.1" ]]; then
manager_server_hostname=$local_ip
fi
- docker pull inlong/dashboard:latest
- docker run -d --name dashboard -e MANAGER_API_ADDRESS=$manager_server_hostname:$manager_server_port -p $inlong_web_port:$docker_inlong_web_port inlong/dashboard
+ docker run -d --name dashboard -e MANAGER_API_ADDRESS=$manager_server_hostname:$manager_server_port -p ${dashboard_port}:80 ${dashboard_docker_image}
}
start_inlong_dataproxy() {
init_compo
- echo "start-up inlong dataproxy ..."
- cd $INLONG_HOME/inlong-dataproxy/bin
- chmod 755 *.sh
- ./dataproxy-start.sh
+ echo "start dataproxy"
+ cd $INLONG_HOME/inlong-dataproxy
+ if [ $mq_type == "pulsar" ]; then
+ ./bin/dataproxy-start.sh pulsar
+ fi
+ if [ $mq_type == "tubemq" ]; then
+ ./bin/dataproxy-start.sh tube
+ fi
+ # wait to start
+ wait_port_to_listen dataproxy ${dataproxy_port}
}
start_inlong_agent() {
init_compo
- echo "start-up inlong agent ..."
- cd $INLONG_HOME/inlong-agent/bin
- ./agent.sh start
-}
-
-start_inlong_sort() {
- echo "start-up inlong sort ..."
- # Judge whether the system has started Fink cluster
- flink_thread=$($JAVA_HOME/bin/jps | grep TaskManagerRunner)
- if [ ! -n "$flink_thread" ]; then
- echo "The system does not start Flink. Please start Flink manually first"
- exit 1
- else
- echo "Currently, the system starts Flink, which is used to process sort tasks"
- fi
-
- cd $INLONG_HOME
- $FLINK_HOME/bin/flink run -c org.apache.inlong.sort.flink.Entrance inlong-sort/sort-dist*.jar \
- --cluster-id $sort_app_name --zookeeper.quorum $zkserver_addr --zookeeper.path.root $cluster_zk_root \
- --source.type $source_type --sink.type $sink_type &
+ echo "start agent"
+ cd $INLONG_HOME/inlong-agent
+ ./bin/agent.sh start
+ # wait to start
+ wait_port_to_listen agent ${agent_port}
+}
+
+# register dataproxy for standalone
+register_service() {
+ curl --header "Content-Type: application/json" --request POST http://localhost:8083/api/inlong/manager/openapi/cluster/save --data '
+ {
+ "name": "default_dataproxy",
+ "type": "DATA_PROXY",
+ "ip": "127.0.0.7",
+ "port": 46801,
+ "mqSetName": "default_set_name",
+ "inCharges": "admin",
+ "creator": "admin"
+ }
+ '
}
# start inlong
start_inlong_all() {
- echo "init components"
+ echo "init configuration"
cd $INLONG_HOME/bin
./init-config.sh
- # start-up message middleware
- echo "Judge the choice of message middleware tubemq or pulsar"
- if [ $source_type == "pulsar" ]; then
- # Judge whether the pulsar cluster is started in the current system
- pulsar_thread=$($JAVA_HOME/bin/jps | egrep 'PulsarBrokerStarter | PulsarStandaloneStarter')
- if [ ! -n "$pulsar_thread" ]; then
- echo "The system does not start the pulsar. Please start the pulsar manually first"
- exit 1
- else
- echo "The current system starts the pulsar, which is used to complete message delivery and storage"
- fi
- else
- #start-up inlong tubemaster
- start_inlong_tubemaster
- sleep 15
- #start-up inlong tubebroker
- start_inlong_tubebroker
- #start-up inlong tubemanager
- start_inlong_tubemanager
- fi
- #start-up inlong audit
- start_inlong_audit
- # start-up inlong manager
+ # start inlong manager
start_inlong_manager
- # start-up inlong dashboard
+ # start inlong audit
+ start_inlong_audit
+ # start inlong dashboard
start_inlong_dashboard
- # start-up inlong dataproxy
+ # start inlong dataproxy
start_inlong_dataproxy
- # start-up inlong agent
+ register_service
+ # start inlong agent
start_inlong_agent
- # start-up inlong sort
- start_inlong_sort
-}
-
-stop_inlong_tubemaster() {
- echo "stop tubemq_master... "
- cd $INLONG_HOME/inlong-tubemq-server/bin
- chmod 755 tubemq.sh
- ./tubemq.sh master stop
-}
-
-stop_inlong_tubebroker() {
- echo "stop tubemq_broker ... "
- cd $INLONG_HOME/inlong-tubemq-server/bin
- chmod 755 tubemq.sh
- ./tubemq.sh broker stop
-}
-
-stop_inlong_tubemanager() {
- echo "stop tubemq_manager ... "
- cd $INLONG_HOME/inlong-tubemq-manager/bin
- ./stop-manager.sh
}
stop_inlong_manager() {
- echo "stop inlong_manager ... "
+ echo "stop manager"
cd $INLONG_HOME/inlong-manager/bin
./shutdown.sh
}
@@ -269,76 +179,41 @@ stop_inlong_dashboard() {
}
stop_inlong_dataproxy() {
- echo "stop dataproxy ... "
+ echo "stop dataproxy"
cd $INLONG_HOME/inlong-dataproxy/bin
- chmod 755 *.sh
./dataproxy-stop.sh
}
stop_inlong_audit() {
- echo "stop audit ... "
+ echo "stop audit"
cd $INLONG_HOME/inlong-audit/bin
./proxy-stop.sh
./store-stop.sh
}
stop_inlong_agent() {
- echo "stop agent ... "
+ echo "stop agent"
cd $INLONG_HOME/inlong-agent/bin
./agent.sh stop
}
-stop_inlong_sort() {
- echo "stop CliFrontend... "
- cli_frontend_thread=$($JAVA_HOME/bin/jps | grep CliFrontend)
- if [ ! -n "$cli_frontend_thread" ]; then
- echo "The system did not running cli_frontend"
- else
- cli_frontend_pid=$(echo $cli_frontend_thread | tr -cd "[0-9]")
- kill -9 $cli_frontend_pid
- echo "cli_frontend stopped "
- fi
- echo "stop sort flink job"
- runjob=$($FLINK_HOME/bin/flink list -r | grep "$sort_app_name")
- OLD_IFS="$IFS"
- IFS=":"
- array=($runjob)
- IFS="$OLD_IFS"
- jobId=$(echo "${array[3]}")
- echo $jobId
- $FLINK_HOME/bin/flink cancel $jobId
-}
-
# stop inlong
stop_inlong_all() {
- #stop inlong tubemq
- if [ $source_type == "tubemq" ]; then
- # stop inlong tubemaster
- stop_inlong_tubemaster
- # stop inlong tubebroker
- stop_inlong_tubebroker
- # stop inlong tubemanager
- stop_inlong_tubemanager
- fi
- # stop inlong manager
- stop_inlong_manager
- # stop inlong website
+ # stop inlong dashboard
stop_inlong_dashboard
- if [ $source_type == "pulsar" ]; then
- # stop inlong audit
- stop_inlong_audit
- fi
# stop inlong dataproxy
stop_inlong_dataproxy
# stop inlong agent
stop_inlong_agent
- # stop inlong sort
- stop_inlong_sort
+ # stop inlong audit
+ stop_inlong_audit
+ # stop inlong manager
+ stop_inlong_manager
}
if [[ "$COMMAND" == start || "$COMMAND" == stop ]]; then
case $SERVICE in
- tubemaster | tubebroker | tubemanager | manager | dashboard | dataproxy | agent | sort | audit)
+ manager | dashboard | dataproxy | agent | audit)
${COMMAND}_inlong_${SERVICE}
;;
standalone)
diff --git a/conf/inlong.conf b/conf/inlong.conf
index 546ea57..e4626b2 100644
--- a/conf/inlong.conf
+++ b/conf/inlong.conf
@@ -14,119 +14,47 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-# Overall configuration of inlong service components
-# source sink (Required)
-###############################################################################
-# Selection of message queue source: tubemq or pulsar
-source_type=tubemq
-# Selection of sort sink: hive or clickhouse
-sink_type=hive
-###############################################################################
-
-# agent
-###############################################################################
-agent_local_ip=127.0.0.1
-# whether to enable prometheus
-agent_prometheus_enable=false
-# whether to enable audit
-agent_audit_enable=true
-###############################################################################
-
-# dataproxy
-###############################################################################
-dataproxy_ip=127.0.0.1
-###############################################################################
-
-# audit proxy
-###############################################################################
-audit_proxys_ip=127.0.0.1
-###############################################################################
-
-###############################################################################
-# mysql(Required) (tubemanager,apache_inlong_manager,apache_inlong_audit)
-###############################################################################
+############## Common Configuration ##############
+# local IP
+local_ip=127.0.0.1
+# message queue, tubemq or pulsar
+mq_type=pulsar
# MySQL service, IP, port, user and password
-spring_datasource_hostname=MYSQL_HOSTNAME
-spring_datasource_port=MYSQL_PORT
-spring_datasource_username=MYSQL_USERNAME
-spring_datasource_password=MYSQL_PASSWORD
-###############################################################################
-
-# zookeeper(Required)
-###############################################################################
-# zookeeper address
-zkserver_addr=127.0.0.1:2181
-###############################################################################
-
-# tubemq (Optional)
-###############################################################################
-# master
-# host address of master, required; must be configured at network card, enabled
-tubemaster_hostname=YOU_LOCAL_IP
-# port that master listens to, optional; default is 8715
-tubemaster_port=8715
-# port that master web console listens to
-tubemaster_webport=8080
-# metaDataPath
-metadata_path=/stage/meta_data
-# This token is used for page configuration, API call, etc
-confmod_authtoken=abc
+spring_datasource_hostname=127.0.0.1
+spring_datasource_port=3306
+spring_datasource_username=root
+spring_datasource_password=inlong
-# broker
-# unique id of broker server, required; when set to 0, TubeMQ will retrieve
-# IP, convert to int and set as brokerID
-broker_id=1
-# hostname of broker server, required
-tubebroker_hostname=YOU_LOCAL_IP
-# port that broker server listens to, optional; default is 8123
-tubebroker_port=8123
-# port that broker web console listens to
-tubebroker_webport=8081
-# path to message files
-primary_path=/stage/msg_data
+############## Pulsar Configuration ##############
+pulsar_service_url=pulsar://127.0.0.1:6650
+pulsar_admin_url=http://127.0.0.1:8080
-# manager(Required)
-# the parameters for init cluster
-tube_manager_ip=YOU_LOCAL_IP
-tube_manager_port=8089
-###############################################################################
+############## TubeMQ Configuration ##############
+tube_master_url=127.0.0.1:8715
+tube_manager_url=127.0.0.1:8080
-# pulsar (Optional)
-###############################################################################
-# Service address of pulsar (Required)
-pulsar_admin_url=http://127.0.0.1:8080
-# Pulsar broker address（Required）
-pulsar_service_url=pulsar://127.0.0.1:6650
-# Default tenant of Pulsar
-pulsar_default_tenant=public
-###############################################################################
+############## Dashboard Configuration ##############
+# dashboard docker image
+dashboard_docker_image=inlong/dashboard:latest
+# dashboard service port
+dashboard_mapping_port=80
-# inlong manager
-###############################################################################
-# manager-web (Required)
+############## Manager Configuration ##############
+# The default value is dev
+spring_profiles_active=dev
+# manager service IP
manager_server_hostname=127.0.0.1
+# manager port
manager_server_port=8083
-# The default configuration file is dev
-spring_profiles_active=dev
-cluster_zk_root=inlong_hive
-# app_name-InLong Sort is cluster-id
-sort_app_name=inlong_app
-###############################################################################
-# inlong website(Required)
-###############################################################################
-# port
-inlong_web_port=80
-# docker port
-docker_inlong_web_port=80
-###############################################################################
+############## DataProxy Configuration ##############
+dataproxy_port=46801
-# flink_home(Required)
-###############################################################################
-FLINK_HOME=/tmp/flink
-###############################################################################
+############## Agent Configuration ##############
+agent_port=8008
-# local_ip(Required)
-###############################################################################
-local_ip=YOU_LOCAL_IP
-###############################################################################
\ No newline at end of file
+############## Audit Configuration ##############
+# audit proxy IP
+audit_proxys_ip=127.0.0.1
+# audit proxy Port
+audit_proxys_port=10081
\ No newline at end of file
diff --git a/docker/docker-compose/sql/tubemanager.sql b/docker/docker-compose/sql/tubemanager.sql
deleted file mode 100644
index d23e3b3..0000000
--- a/docker/docker-compose/sql/tubemanager.sql
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-SET NAMES utf8mb4;
-SET FOREIGN_KEY_CHECKS = 0;
-
--- ----------------------------
--- database for TubeMQ Manager
--- ----------------------------
-CREATE DATABASE IF NOT EXISTS tubemanager;
diff --git a/inlong-manager/manager-web/bin/startup.sh b/inlong-manager/manager-web/bin/startup.sh
index dbaee3b..3e54b00 100755
--- a/inlong-manager/manager-web/bin/startup.sh
+++ b/inlong-manager/manager-web/bin/startup.sh
@@ -71,9 +71,6 @@ LOG_STARTUP_PATH="${LOG_DIR}/startup.log"
NOW=$(date +'%Y-%m-%m-%H-%M-%S')
NOW_PRETTY=$(date +'%Y-%m-%m %H:%M:%S')
-# Startup log
-STARTUP_LOG="================================================ ${NOW_PRETTY} ================================================\n"
-
# If the logs folder does not exist, create the folder
if [ ! -d "${LOG_DIR}" ]; then
mkdir "${LOG_DIR}"
@@ -114,38 +111,5 @@ JAVA_OPT="${JAVA_OPT} -XX:+IgnoreUnrecognizedVMOptions -XX:+UseConcMarkSweepGC -
# Remote debugger
#JAVA_OPT="${JAVA_OPT} -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8081"
-#=======================================================
-# Append command startup related logs to the log file
-#=======================================================
-
-# Output project name
-STARTUP_LOG="${STARTUP_LOG}application name: ${APPLICATION}\n"
-# Output jar package name
-STARTUP_LOG="${STARTUP_LOG}application jar name: ${APPLICATION_JAR}\n"
-# Output project root directory
-STARTUP_LOG="${STARTUP_LOG}application root path: ${BASE_PATH}\n"
-# Output project bin path
-STARTUP_LOG="${STARTUP_LOG}application bin path: ${BIN_PATH}\n"
-# Output project config path
-STARTUP_LOG="${STARTUP_LOG}application config path: ${CONFIG_DIR}\n"
-# Print log path
-STARTUP_LOG="${STARTUP_LOG}application log path: ${LOG_DIR}\n"
-# Print JVM configuration
-STARTUP_LOG="${STARTUP_LOG}application JAVA_OPT: ${JAVA_OPT}\n"
-
-# Print start command
-STARTUP_LOG="${STARTUP_LOG}application startup command: nohup java ${JAVA_OPT} -Dlog4j2.formatMsgNoLookups=true -Dlog4j.formatMsgNoLookups=true -cp ${CLASSPATH} ${MAIN_CLASS} 1>${LOG_FILE} 2>${LOG_DIR}/error.log &\n"
-
-#======================================================================
# Execute the startup command: start the project in the background, and output the log to the logs folder under the project root directory
-#======================================================================
-nohup java ${JAVA_OPT} -Dlog4j2.formatMsgNoLookups=true -Dlog4j.formatMsgNoLookups=true -cp ${CLASSPATH} ${MAIN_CLASS} 1>${LOG_FILE} 2>${LOG_DIR}/error.log &
-
-# Process ID
-PID="$!"
-STARTUP_LOG="${STARTUP_LOG}application pid: ${PID}\n"
-
-# The startup log is appended to the startup log file
-echo -e ${STARTUP_LOG} >>${LOG_STARTUP_PATH}
-# Print startup log
-echo -e ${STARTUP_LOG}
+nohup java ${JAVA_OPT} -Dlog4j2.formatMsgNoLookups=true -Dlog4j.formatMsgNoLookups=true -cp ${CLASSPATH} ${MAIN_CLASS} 1>${LOG_FILE} 2>${LOG_DIR}/error.log &
\ No newline at end of file
diff --git a/inlong-tubemq/tubemq-docker/tubemq-manager/manager-docker.sh b/inlong-tubemq/tubemq-docker/tubemq-manager/manager-docker.sh
index c9ab444..b795930 100644
--- a/inlong-tubemq/tubemq-docker/tubemq-manager/manager-docker.sh
+++ b/inlong-tubemq/tubemq-docker/tubemq-manager/manager-docker.sh
@@ -25,7 +25,7 @@ spring.mvc.pathmatch.matching-strategy = ANT_PATH_MATCHER
topic.config.schedule=0/5 * * * * ?
broker.reload.schedule=0/5 * * * * ?
# mysql configuration for manager
-spring.datasource.url=jdbc:mysql://$MYSQL_HOST:$MYSQL_PORT/tubemanager?useSSL=false
+spring.datasource.url=jdbc:mysql://$MYSQL_HOST:$MYSQL_PORT/apache_inlong_tubemq?useSSL=false
spring.datasource.username=$MYSQL_USER
spring.datasource.password=$MYSQL_PASSWD
# server port
diff --git a/inlong-tubemq/tubemq-manager/sql/apache_tube_manager.sql b/inlong-tubemq/tubemq-manager/sql/apache_tube_manager.sql
index 7a7cc2a..a7d4e47 100644
--- a/inlong-tubemq/tubemq-manager/sql/apache_tube_manager.sql
+++ b/inlong-tubemq/tubemq-manager/sql/apache_tube_manager.sql
@@ -21,7 +21,7 @@ SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
-- database for TubeMQ Manager
-- ----------------------------
-CREATE DATABASE IF NOT EXISTS tubemanager;
+CREATE DATABASE IF NOT EXISTS apache_inlong_tubemq;
-- ----------------------------
-- Table structure for broker
diff --git a/inlong-tubemq/tubemq-manager/src/main/resources/application.properties b/inlong-tubemq/tubemq-manager/src/main/resources/application.properties
index 841c97a..f042c6e 100644
--- a/inlong-tubemq/tubemq-manager/src/main/resources/application.properties
+++ b/inlong-tubemq/tubemq-manager/src/main/resources/application.properties
@@ -21,7 +21,7 @@ spring.mvc.pathmatch.matching-strategy = ANT_PATH_MATCHER
topic.config.schedule=0/5 * * * * ?
broker.reload.schedule=0/5 * * * * ?
# mysql configuration for manager
-# spring.datasource.url=jdbc:mysql://mysql_ip:mysql_port/tubemanager?useSSL=false&allowPublicKeyRetrieval=true
+# spring.datasource.url=jdbc:mysql://mysql_ip:mysql_port/apache_inlong_tubemq?useSSL=false&allowPublicKeyRetrieval=true
# spring.datasource.username=mysql_username
# spring.datasource.password=mysql_password
# server port