You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ke...@apache.org on 2021/12/01 06:08:07 UTC
[dolphinscheduler] branch 2.0.1-prepare updated: [2.0.0-chery-pick #7003]Fix shell tools for database schema cannot run (#7087)
This is an automated email from the ASF dual-hosted git repository.
kerwin pushed a commit to branch 2.0.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/2.0.1-prepare by this push:
new b3e8704 [2.0.0-chery-pick #7003]Fix shell tools for database schema cannot run (#7087)
b3e8704 is described below
commit b3e8704194f62f96404277040e93350451048a67
Author: Kirs <ac...@163.com>
AuthorDate: Wed Dec 1 14:08:02 2021 +0800
[2.0.0-chery-pick #7003]Fix shell tools for database schema cannot run (#7087)
PR #7003
ISSUE #6998
---
docker/build/hooks/build | 42 ++----
docker/build/startup-init-conf.sh | 2 +
.../common/utils/SchemaUtilsTest.java | 117 ----------------
.../dao/datasource/SpringConnectionFactory.java | 155 ++-------------------
.../dolphinscheduler/dao/upgrade/SchemaUtils.java | 118 ++++++++++++++++
.../dolphinscheduler/dao/upgrade/UpgradeDao.java | 5 +-
.../apache/dolphinscheduler/dao/BaseDaoTest.java | 1 +
.../dao/upgrade/SchemaUtilsTest.java | 60 ++++++++
script/create-dolphinscheduler.sh | 6 +-
script/dolphinscheduler-daemon.sh | 16 ++-
script/scp-hosts.sh | 2 +
script/upgrade-dolphinscheduler.sh | 6 +-
12 files changed, 223 insertions(+), 307 deletions(-)
diff --git a/docker/build/hooks/build b/docker/build/hooks/build
index 70ea260..1e6a965 100755
--- a/docker/build/hooks/build
+++ b/docker/build/hooks/build
@@ -18,41 +18,17 @@
set -e
-echo "------ dolphinscheduler start - build -------"
-printenv
+ROOT_DIR=$(dirname "$0")/../../..
+MVN="$ROOT_DIR"/mvnw
+VERSION=$("$MVN" -q -DforceStdout -N org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version)
-if [ -z "${VERSION}" ]
-then
- echo "set default environment variable [VERSION]"
- export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
-fi
+DOCKER_REPO=${DOCKER_REPO:-"apache/dolphinscheduler"}
+TAG=${TAG:-"$VERSION"}
-if [ "${DOCKER_REPO}x" = "x" ]
-then
- echo "set default environment variable [DOCKER_REPO]"
- export DOCKER_REPO='apache/dolphinscheduler'
-fi
+echo "Building Docker image as: $DOCKER_REPO:$TAG"
-echo "Version: $VERSION"
-echo "Repo: $DOCKER_REPO"
+"$MVN" -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-echo -e "Current Directory is $(pwd)\n"
+cp "$ROOT_DIR"/dolphinscheduler-dist/target/apache-dolphinscheduler-$VERSION-bin.tar.gz "$ROOT_DIR"/docker/build/
-# maven package(Project Directory)
-echo -e "./mvnw -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120"
-./mvnw -B clean package -Prelease -Dmaven.test.skip=true -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-
-# mv dolphinscheduler-bin.tar.gz file to docker/build directory
-echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/\n"
-mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-${VERSION}-bin.tar.gz $(pwd)/docker/build/
-
-# docker build
-BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/"
-echo -e "$BUILD_COMMAND\n"
-if (docker info 2> /dev/null | grep -i "ERROR"); then
- sudo $BUILD_COMMAND
-else
- $BUILD_COMMAND
-fi
-
-echo "------ dolphinscheduler end - build -------"
+docker build --build-arg VERSION=$VERSION -t $DOCKER_REPO:$TAG "$ROOT_DIR"/docker/build/
diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh
index 32b71bf..ad61829 100755
--- a/docker/build/startup-init-conf.sh
+++ b/docker/build/startup-init-conf.sh
@@ -37,6 +37,8 @@ echo "init env variables"
[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_URL="jdbc:${DATABASE_TYPE}://${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DATABASE}?${DATABASE_PARAMS}"
[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_USERNAME=${DATABASE_USERNAME}
[ "h2" == ${DATABASE_TYPE} ] || export SPRING_DATASOURCE_PASSWORD=${DATABASE_PASSWORD}
+# export SPRING_DATASOURCE_DRIVER_CLASS_NAME=${DATABASE_DRIVER}
+
#============================================================================
# Registry
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java
deleted file mode 100644
index b385523..0000000
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.common.utils;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.Arrays;
-import java.util.List;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({ LoggerFactory.class, FileUtils.class })
-public class SchemaUtilsTest {
-
- @Test
- public void testReplaceBlank() {
- Assert.assertEquals("abc", SchemaUtils.replaceBlank(" abc"));
- Assert.assertEquals("abc", SchemaUtils.replaceBlank("abc "));
- Assert.assertEquals("abc", SchemaUtils.replaceBlank("a b c"));
- Assert.assertEquals("abc", SchemaUtils.replaceBlank("a b c"));
- Assert.assertEquals("", SchemaUtils.replaceBlank(" "));
- Assert.assertEquals("", SchemaUtils.replaceBlank(null));
- Assert.assertEquals("我怕的你", SchemaUtils.replaceBlank("我怕的 你"));
- }
-
- @Test
- public void testGetSoftVersion() {
- // file not found
- try {
- SchemaUtils.getSoftVersion();
- } catch (RuntimeException e) {
- Assert.assertEquals("Failed to get the product version description file. The file could not be found",
- e.getMessage());
- }
-
- // file exists, fmt is invalid
- FileUtils.writeContent2File("32432423", "sql/soft_version");
- Assert.assertEquals("32432423", SchemaUtils.getSoftVersion());
- }
-
- @Test
- public void testIsAGreatVersion() {
- // param is null
- try {
- SchemaUtils.isAGreatVersion(null, null);
- } catch (RuntimeException e) {
- Assert.assertEquals("schemaVersion or version is empty", e.getMessage());
- }
-
- // param is ""
- try {
- SchemaUtils.isAGreatVersion("", "");
- } catch (RuntimeException e) {
- Assert.assertEquals("schemaVersion or version is empty", e.getMessage());
- }
- Assert.assertFalse(SchemaUtils.isAGreatVersion("1", "1"));
- Assert.assertTrue(SchemaUtils.isAGreatVersion("2", "1"));
- Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1"));
- Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1.0.1"));
- Assert.assertFalse(SchemaUtils.isAGreatVersion("1.1", "1.2"));
- Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1.1", "1.1"));
- Assert.assertTrue(SchemaUtils.isAGreatVersion("10.1.1", "1.01.100"));
- try {
- SchemaUtils.isAGreatVersion("10.1.1", ".1");
- } catch (Exception e) {
- Assert.assertNotNull(e);
- }
- try {
- SchemaUtils.isAGreatVersion("a.1.1", "b.1");
- } catch (Exception e) {
- Assert.assertNotNull(e);
- }
- }
-
- @Test
- public void testGetAllSchemaList() {
- //normal
- PowerMockito.mockStatic(FileUtils.class);
- File[] files = new File[4];
- files[0] = new File("sql/upgrade/1.2.0_schema");
- files[1] = new File("sql/upgrade/1.0.1_schema");
- files[2] = new File("sql/upgrade/1.0.2_schema");
- files[3] = new File("sql/upgrade/1.1.0_schema");
- PowerMockito.when(FileUtils.getAllDir("sql/upgrade")).thenReturn(files);
- List<String> real = SchemaUtils.getAllSchemaList();
- List<String> expect = Arrays.asList("1.0.1_schema", "1.0.2_schema",
- "1.1.0_schema", "1.2.0_schema");
- boolean result = org.apache.commons.collections.CollectionUtils.isEqualCollection(real, expect);
- Assert.assertTrue(result);
-
- //normal
- files = new File[0];
- PowerMockito.when(FileUtils.getAllDir("sql/upgrade")).thenReturn(files);
- real = SchemaUtils.getAllSchemaList();
- Assert.assertNull(real);
- }
-}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
index 29cbefa..e44ac85 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
@@ -17,160 +17,38 @@
package org.apache.dolphinscheduler.dao.datasource;
-import static org.apache.dolphinscheduler.common.Constants.DATASOURCE_PROPERTIES;
-
-import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.enums.ProfileType;
-import org.apache.dolphinscheduler.common.utils.PropertyUtils;
-
-import org.apache.commons.lang.StringUtils;
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.core.MybatisConfiguration;
+import com.baomidou.mybatisplus.core.config.GlobalConfig;
+import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor;
+import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
import org.apache.ibatis.mapping.DatabaseIdProvider;
import org.apache.ibatis.mapping.VendorDatabaseIdProvider;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.type.JdbcType;
-
-import java.sql.SQLException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
-
-import javax.annotation.PostConstruct;
-import javax.sql.DataSource;
-
-import org.h2.Driver;
import org.mybatis.spring.SqlSessionTemplate;
-import org.mybatis.spring.annotation.MapperScan;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
-import org.springframework.core.env.Environment;
-import org.springframework.core.io.FileSystemResourceLoader;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.core.io.support.ResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
-import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
-import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType;
-import com.baomidou.mybatisplus.annotation.IdType;
-import com.baomidou.mybatisplus.core.MybatisConfiguration;
-import com.baomidou.mybatisplus.core.config.GlobalConfig;
-import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor;
-import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
-import com.google.common.collect.Lists;
-import com.zaxxer.hikari.HikariDataSource;
+import javax.sql.DataSource;
+import java.util.Properties;
-/**
- * data source connection factory
- */
@Configuration
-@MapperScan("org.apache.dolphinscheduler.*.mapper")
public class SpringConnectionFactory {
-
- private static final Logger logger = LoggerFactory.getLogger(SpringConnectionFactory.class);
-
- @Autowired
- private Environment environment;
-
- private static final AtomicBoolean H2_INITIALIZED = new AtomicBoolean(false);
-
- @PostConstruct
- public void init() {
- String datasourceProfile = getSpringActiveProfile().stream()
- .filter(ProfileType.DATASOURCE_PROFILE::contains)
- .findFirst()
- .orElse("");
- if (StringUtils.isEmpty(datasourceProfile) || ProfileType.MYSQL.equals(datasourceProfile)) {
- // default load datasource.properties
- PropertyUtils.loadPropertyFile(DATASOURCE_PROPERTIES.replace("-%s", ""));
- } else {
- // load datasource-{spring.profiles.active}.properties
- PropertyUtils.loadPropertyFile(String.format(DATASOURCE_PROPERTIES, datasourceProfile));
- }
- }
-
- /**
- * pagination interceptor
- *
- * @return pagination interceptor
- */
@Bean
public PaginationInterceptor paginationInterceptor() {
return new PaginationInterceptor();
}
- /**
- * get the data source
- *
- * @return dataSource
- */
- @Bean(destroyMethod = "", name = "datasource")
- public DataSource dataSource() throws SQLException {
- String driverClassName = PropertyUtils.getString(Constants.SPRING_DATASOURCE_DRIVER_CLASS_NAME);
- if (Driver.class.getName().equals(driverClassName)) {
- initializeH2Datasource();
- }
-
- HikariDataSource dataSource = new HikariDataSource();
-
- dataSource.setDriverClassName(driverClassName);
- dataSource.setJdbcUrl(PropertyUtils.getString(Constants.SPRING_DATASOURCE_URL));
- dataSource.setUsername(PropertyUtils.getString(Constants.SPRING_DATASOURCE_USERNAME));
- dataSource.setPassword(PropertyUtils.getString(Constants.SPRING_DATASOURCE_PASSWORD));
- dataSource.setConnectionTestQuery(PropertyUtils.getString(Constants.SPRING_DATASOURCE_VALIDATION_QUERY, "SELECT 1"));
-
- dataSource.setMinimumIdle(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MIN_IDLE, 5));
- dataSource.setMaximumPoolSize(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_ACTIVE, 50));
- dataSource.setConnectionTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_CONNECTION_TIMEOUT, 30000));
- dataSource.setIdleTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_IDLE_TIMEOUT, 600000));
- dataSource.setMaxLifetime(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_MAX_LIFE_TIME, 1800000));
- dataSource.setValidationTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_VALIDATION_TIMEOUT, 5000));
- dataSource.setLeakDetectionThreshold(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_LEAK_DETECTION_THRESHOLD, 0));
- dataSource.setInitializationFailTimeout(PropertyUtils.getInt(Constants.SPRING_DATASOURCE_INITIALIZATION_FAIL_TIMEOUT, 1));
-
- dataSource.setAutoCommit(PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_IS_AUTOCOMMIT, true));
-
- dataSource.addDataSourceProperty(Constants.CACHE_PREP_STMTS, PropertyUtils.getBoolean(Constants.SPRING_DATASOURCE_CACHE_PREP_STMTS, true));
- dataSource.addDataSourceProperty(Constants.PREP_STMT_CACHE_SIZE, PropertyUtils.getInt(Constants.SPRING_DATASOURCE_PREP_STMT_CACHE_SIZE, 250));
- dataSource.addDataSourceProperty(Constants.PREP_STMT_CACHE_SQL_LIMIT, PropertyUtils.getInt(Constants.SPRING_DATASOURCE_PREP_STMT_CACHE_SQL_LIMIT, 2048));
-
- logger.info("Initialize DataSource DataSource success");
- return dataSource;
- }
-
- private void initializeH2Datasource() {
- if (H2_INITIALIZED.compareAndSet(false, true)) {
- EmbeddedDatabaseBuilder embeddedDatabaseBuilder = new EmbeddedDatabaseBuilder(new FileSystemResourceLoader());
- embeddedDatabaseBuilder
- .setType(EmbeddedDatabaseType.H2)
- .setScriptEncoding(Constants.UTF_8)
- .setName("dolphinscheduler;MODE=MySQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1")
- .addScript(PropertyUtils.getString("spring.datasource.sql.schema", "classpath:sql/dolphinscheduler_h2.sql"))
- .build();
- logger.info("Initialize H2 DataSource success");
- }
- }
-
- /**
- * * get transaction manager
- *
- * @return DataSourceTransactionManager
- */
@Bean
public DataSourceTransactionManager transactionManager(DataSource dataSource) {
return new DataSourceTransactionManager(dataSource);
}
- /**
- * * get sql session factory
- *
- * @return sqlSessionFactory
- * @throws Exception sqlSessionFactory exception
- */
@Bean
public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Exception {
MybatisConfiguration configuration = new MybatisConfiguration();
@@ -179,6 +57,7 @@ public class SpringConnectionFactory {
configuration.setCallSettersOnNulls(true);
configuration.setJdbcTypeForNull(JdbcType.NULL);
configuration.addInterceptor(paginationInterceptor());
+ configuration.setGlobalConfig(new GlobalConfig().setBanner(false));
MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new MybatisSqlSessionFactoryBean();
sqlSessionFactoryBean.setConfiguration(configuration);
sqlSessionFactoryBean.setDataSource(dataSource);
@@ -196,11 +75,6 @@ public class SpringConnectionFactory {
return sqlSessionFactoryBean.getObject();
}
- /**
- * get sql session
- *
- * @return SqlSession
- */
@Bean
public SqlSession sqlSession(SqlSessionFactory sqlSessionFactory) {
return new SqlSessionTemplate(sqlSessionFactory);
@@ -216,17 +90,4 @@ public class SpringConnectionFactory {
databaseIdProvider.setProperties(properties);
return databaseIdProvider;
}
-
- /**
- * Get spring active profile, which will be set by -Dspring.profiles.active=api， or in application.xml
- *
- * @return
- */
- private List<String> getSpringActiveProfile() {
- if (environment != null) {
- return Lists.newArrayList(environment.getActiveProfiles());
- }
- String property = System.getProperty("spring.profiles.active", "");
- return Arrays.stream(property.split(",")).map(String::trim).collect(Collectors.toList());
- }
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java
new file mode 100644
index 0000000..83acb75
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.upgrade;
+
+import com.google.common.base.Strings;
+import org.apache.commons.lang.StringUtils;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.core.io.ClassPathResource;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Metadata related common classes
+ */
+public class SchemaUtils {
+ private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class);
+
+ private SchemaUtils() {
+ throw new UnsupportedOperationException("Construct SchemaUtils");
+ }
+
+ public static List<String> getAllSchemaList() throws IOException {
+ final File[] schemaDirArr = new ClassPathResource("sql/upgrade").getFile().listFiles();
+
+ if (schemaDirArr == null || schemaDirArr.length == 0) {
+ return Collections.emptyList();
+ }
+
+ return Arrays.stream(schemaDirArr).map(File::getName).sorted((o1, o2) -> {
+ try {
+ String version1 = o1.split("_")[0];
+ String version2 = o2.split("_")[0];
+
+ if (version1.equals(version2)) {
+ return 0;
+ }
+
+ if (SchemaUtils.isAGreatVersion(version1, version2)) {
+ return 1;
+ }
+
+ return -1;
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new RuntimeException(e);
+ }
+ }).collect(Collectors.toList());
+ }
+
+ /**
+ * Determine whether schemaVersion is higher than version
+ *
+ * @param schemaVersion schema version
+ * @param version version
+ * @return Determine whether schemaVersion is higher than version
+ */
+ public static boolean isAGreatVersion(String schemaVersion, String version) {
+ if (StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) {
+ throw new RuntimeException("schemaVersion or version is empty");
+ }
+
+ String[] schemaVersionArr = schemaVersion.split("\\.");
+ String[] versionArr = version.split("\\.");
+ int arrLength = Math.min(schemaVersionArr.length, versionArr.length);
+ for (int i = 0; i < arrLength; i++) {
+ if (Integer.parseInt(schemaVersionArr[i]) > Integer.parseInt(versionArr[i])) {
+ return true;
+ } else if (Integer.parseInt(schemaVersionArr[i]) < Integer.parseInt(versionArr[i])) {
+ return false;
+ }
+ }
+
+ // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number
+ return schemaVersionArr.length > versionArr.length;
+ }
+
+ /**
+ * Gets the current software version number of the system
+ *
+ * @return current software version
+ */
+ public static String getSoftVersion() throws IOException {
+ final ClassPathResource softVersionFile = new ClassPathResource("sql/soft_version");
+ String softVersion;
+ try {
+ softVersion = FileUtils.readFile2Str(softVersionFile.getInputStream());
+ softVersion = Strings.nullToEmpty(softVersion).replaceAll("\\s+|\r|\n", "");
+ } catch (FileNotFoundException e) {
+ logger.error(e.getMessage(), e);
+ throw new RuntimeException("Failed to get the product version description file. The file could not be found", e);
+ }
+ return softVersion;
+ }
+
+}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
index 2f9fb5d..161f8b7 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
@@ -28,7 +28,10 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
-import org.apache.dolphinscheduler.common.utils.*;
+import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
+import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.ScriptRunner;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java
index ddb87d5..5838ad7 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/BaseDaoTest.java
@@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.dao;
import org.apache.dolphinscheduler.common.enums.ProfileType;
+
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.Rollback;
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java
new file mode 100644
index 0000000..b505709
--- /dev/null
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.upgrade;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SchemaUtilsTest {
+
+ @Test
+ public void testIsAGreatVersion() {
+ // param is null
+ try {
+ SchemaUtils.isAGreatVersion(null, null);
+ } catch (RuntimeException e) {
+ Assert.assertEquals("schemaVersion or version is empty", e.getMessage());
+ }
+
+ // param is ""
+ try {
+ SchemaUtils.isAGreatVersion("", "");
+ } catch (RuntimeException e) {
+ Assert.assertEquals("schemaVersion or version is empty", e.getMessage());
+ }
+ Assert.assertFalse(SchemaUtils.isAGreatVersion("1", "1"));
+ Assert.assertTrue(SchemaUtils.isAGreatVersion("2", "1"));
+ Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1"));
+ Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1.0.1"));
+ Assert.assertFalse(SchemaUtils.isAGreatVersion("1.1", "1.2"));
+ Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1.1", "1.1"));
+ Assert.assertTrue(SchemaUtils.isAGreatVersion("10.1.1", "1.01.100"));
+ try {
+ SchemaUtils.isAGreatVersion("10.1.1", ".1");
+ Assert.fail("Should fail");
+ } catch (Exception ignored) {
+ // This is expected
+ }
+ try {
+ SchemaUtils.isAGreatVersion("a.1.1", "b.1");
+ Assert.fail("Should fail");
+ } catch (Exception ignored) {
+ // This is expected
+ }
+ }
+}
diff --git a/script/create-dolphinscheduler.sh b/script/create-dolphinscheduler.sh
index 9ca9499..8591277 100755
--- a/script/create-dolphinscheduler.sh
+++ b/script/create-dolphinscheduler.sh
@@ -22,16 +22,20 @@ DOLPHINSCHEDULER_HOME=$BIN_DIR/..
export JAVA_HOME=$JAVA_HOME
+export DATABASE_TYPE=${DATABASE_TYPE:-"h2"}
+export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"}
+export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},${DATABASE_TYPE}"
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
+export DOLPHINSCHEDULER_SQL_DIR=$DOLPHINSCHEDULER_HOME/sql
export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export STOP_TIMEOUT=5
CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.CreateDolphinScheduler
-exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
+exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_SQL_DIR:$DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
cd $DOLPHINSCHEDULER_HOME
$JAVA_HOME/bin/java $exec_command
diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh
index 41a7a15..64d6b83 100755
--- a/script/dolphinscheduler-daemon.sh
+++ b/script/dolphinscheduler-daemon.sh
@@ -38,8 +38,6 @@ DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd`
source /etc/profile
source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh"
-export JAVA_HOME=$JAVA_HOME
-#export JAVA_HOME=/opt/soft/jdk
export HOSTNAME=`hostname`
export DOLPHINSCHEDULER_PID_DIR=$DOLPHINSCHEDULER_HOME/pid
@@ -61,36 +59,40 @@ cd $DOLPHINSCHEDULER_HOME
export DOLPHINSCHEDULER_OPTS="-server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:$DOLPHINSCHEDULER_LOG_DIR/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm $DOLPHINSCHEDULER_OPTS"
+export DATABASE_TYPE=${DATABASE_TYPE:-"h2"}
+export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"}
+
if [ "$command" = "api-server" ]; then
- LOG_FILE="-Dlogging.config=classpath:logback-api.xml -Dspring.profiles.active=api"
+ LOG_FILE="-Dlogging.config=classpath:logback-api.xml"
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $API_SERVER_OPTS"
+ export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},api,${DATABASE_TYPE}"
elif [ "$command" = "master-server" ]; then
LOG_FILE="-Dlogging.config=classpath:logback-master.xml"
CLASS=org.apache.dolphinscheduler.server.master.MasterServer
HEAP_OPTS="-Xms4g -Xmx4g -Xmn2g"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $MASTER_SERVER_OPTS"
+ export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},master,${DATABASE_TYPE}"
elif [ "$command" = "worker-server" ]; then
LOG_FILE="-Dlogging.config=classpath:logback-worker.xml"
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
HEAP_OPTS="-Xms2g -Xmx2g -Xmn1g"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $WORKER_SERVER_OPTS"
+ export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},worker,${DATABASE_TYPE}"
elif [ "$command" = "alert-server" ]; then
LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml"
CLASS=org.apache.dolphinscheduler.alert.AlertServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $ALERT_SERVER_OPTS"
+ export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},alert,${DATABASE_TYPE}"
elif [ "$command" = "logger-server" ]; then
CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
HEAP_OPTS="-Xms1g -Xmx1g -Xmn512m"
export DOLPHINSCHEDULER_OPTS="$HEAP_OPTS $DOLPHINSCHEDULER_OPTS $LOGGER_SERVER_OPTS"
-elif [ "$command" = "zookeeper-server" ]; then
- #note: this command just for getting a quick experience，not recommended for production. this operation will start a standalone zookeeper server
- LOG_FILE="-Dlogback.configurationFile=classpath:logback-zookeeper.xml"
- CLASS=org.apache.dolphinscheduler.service.zk.ZKServer
elif [ "$command" = "standalone-server" ]; then
CLASS=org.apache.dolphinscheduler.server.StandaloneServer
+ export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},standalone,${DATABASE_TYPE}"
else
echo "Error: No command named '$command' was found."
exit 1
diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh
index 8def4d1..da08446 100755
--- a/script/scp-hosts.sh
+++ b/script/scp-hosts.sh
@@ -29,6 +29,7 @@ fi
declare -A workersGroupMap=()
workersGroup=(${workers//,/ })
+# shellcheck disable=SC2068
for workerGroup in ${workersGroup[@]}
do
echo $workerGroup;
@@ -39,6 +40,7 @@ done
hostsArr=(${ips//,/ })
+# shellcheck disable=SC2068
for host in ${hostsArr[@]}
do
diff --git a/script/upgrade-dolphinscheduler.sh b/script/upgrade-dolphinscheduler.sh
index 42ba839..a8ef4e7 100755
--- a/script/upgrade-dolphinscheduler.sh
+++ b/script/upgrade-dolphinscheduler.sh
@@ -22,16 +22,20 @@ DOLPHINSCHEDULER_HOME=$BIN_DIR/..
export JAVA_HOME=$JAVA_HOME
+export DATABASE_TYPE=${DATABASE_TYPE:-"h2"}
+export SPRING_PROFILES_ACTIVE=${SPRING_PROFILES_ACTIVE:-"default"}
+export SPRING_PROFILES_ACTIVE="${SPRING_PROFILES_ACTIVE},${DATABASE_TYPE}"
export DOLPHINSCHEDULER_CONF_DIR=$DOLPHINSCHEDULER_HOME/conf
export DOLPHINSCHEDULER_LIB_JARS=$DOLPHINSCHEDULER_HOME/lib/*
+export DOLPHINSCHEDULER_SQL_DIR=$DOLPHINSCHEDULER_HOME/sql
export DOLPHINSCHEDULER_OPTS="-server -Xms64m -Xmx64m -Xss512k -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=64m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
export STOP_TIMEOUT=5
CLASS=org.apache.dolphinscheduler.dao.upgrade.shell.UpgradeDolphinScheduler
-exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
+exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_SQL_DIR:$DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS"
cd $DOLPHINSCHEDULER_HOME
$JAVA_HOME/bin/java $exec_command