You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/02/27 08:00:50 UTC
[kylin] 10/34: KYLIN-5449 refactor common-service kylin-tool
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c9769b0ca2bfaf2b51d1291327e0ab7681f3a6c1
Author: qianhao.zhou <z....@gmail.com>
AuthorDate: Sun Dec 25 12:41:04 2022 +0800
KYLIN-5449 refactor common-service kylin-tool
* refactor common-service kylin-tool
* fix UT
* fix UT
* fix UT
* fix code smell
* fix code smell
* fix code smell
* remove systools pom file
Co-authored-by: qhzhou <qi...@kyligence.io>
---
.../kylin/rest/controller/NSystemController.java | 10 +-
src/common-service/pom.xml | 11 +-
.../org/apache/kylin/helper/HelperConstants.java} | 18 +-
.../apache/kylin/helper/MetadataToolHelper.java} | 568 +++++++++------------
.../org/apache/kylin/helper/RoutineToolHelper.java | 116 +++++
.../kylin/helper/UpdateUserAclToolHelper.java | 80 +++
.../kylin/rest/security/AdminUserAspect.java | 16 +-
.../kylin/rest/service/MetadataBackupService.java | 49 +-
.../apache/kylin/rest/service/ProjectService.java | 2 +-
.../apache/kylin/rest/service/SystemService.java | 36 +-
.../apache/kylin/rest/service/UserAclService.java | 8 +-
.../org/apache/kylin/tool/HDFSMetadataTool.java | 4 +
.../apache/kylin/tool/constant/DiagTypeEnum.java | 0
.../org/apache/kylin/tool/constant/StageEnum.java | 0
.../org/apache/kylin/tool/daemon/CheckResult.java | 0
.../apache/kylin/tool/daemon/CheckStateEnum.java | 0
.../apache/kylin/tool/daemon/HealthChecker.java | 0
.../kylin/tool/daemon/KapGuardianHATask.java | 0
.../kylin/tool/daemon/ServiceOpLevelEnum.java | 0
.../java/org/apache/kylin/tool/daemon/Worker.java | 4 +-
.../tool/daemon/checker/AbstractHealthChecker.java | 0
.../tool/daemon/checker/FullGCDurationChecker.java | 0
.../tool/daemon/checker/KEProcessChecker.java | 0
.../kylin/tool/daemon/checker/KEStatusChecker.java | 10 +-
.../kylin/tool/garbage/ExecutableCleaner.java | 5 +-
.../apache/kylin/tool/garbage/GarbageCleaner.java | 4 -
.../apache/kylin/tool/garbage/IndexCleaner.java | 0
.../apache/kylin/tool/garbage/MetadataCleaner.java | 4 +-
.../apache/kylin/tool/garbage/SnapshotCleaner.java | 5 +-
.../kylin/tool/garbage/SourceUsageCleaner.java | 0
.../apache/kylin/tool/garbage/StorageCleaner.java | 88 ++--
.../tool/kerberos/DelegationTokenManager.java | 0
.../kylin/tool/kerberos/KerberosLoginUtil.java | 164 ++----
.../java/org/apache/kylin/tool/util/LdapUtils.java | 0
.../util/ProjectTemporaryTableCleanerHelper.java | 0
.../java/org/apache/kylin/tool/util/ToolUtil.java | 25 +-
.../kylin/rest/service/LdapUserServiceTest.java | 11 +-
.../kylin/rest/service/OpenUserServiceTest.java | 7 +-
.../ProjectTemporaryTableCleanerHelperTest.java | 0
.../org/apache/kylin/tool/util/ToolUtilTest.java | 0
.../java/org/apache/kylin/common/util/Pair.java | 2 +-
.../org/apache/kylin/tool/util/HashFunction.java | 0
.../apache/kylin/tool/util/HashFunctionTest.java | 0
.../kylin/job/execution/NExecutableManager.java | 2 -
src/core-metadata/pom.xml | 4 +
.../kylin/metrics/HdfsCapacityMetricsTest.java | 3 +
.../apache/kylin/rest/service/ScheduleService.java | 20 +-
src/kylin-it/pom.xml | 4 +
.../rest/controller/NMetaStoreController.java | 3 +-
.../kylin/rest/service/MetaStoreService.java | 18 +-
.../org/apache/kylin/tool/bisync/BISyncModel.java | 0
.../kylin/tool/bisync/BISyncModelConverter.java | 0
.../org/apache/kylin/tool/bisync/BISyncTool.java | 0
.../org/apache/kylin/tool/bisync/SyncContext.java | 0
.../apache/kylin/tool/bisync/SyncModelBuilder.java | 0
.../apache/kylin/tool/bisync/model/ColumnDef.java | 0
.../kylin/tool/bisync/model/JoinTreeNode.java | 3 +-
.../apache/kylin/tool/bisync/model/MeasureDef.java | 0
.../apache/kylin/tool/bisync/model/SyncModel.java | 0
.../bisync/tableau/TableauDataSourceConverter.java | 2 +-
.../bisync/tableau/TableauDatasourceModel.java | 4 +-
.../tool/bisync/tableau/datasource/Aliases.java | 0
.../tool/bisync/tableau/datasource/DrillPath.java | 13 -
.../tool/bisync/tableau/datasource/DrillPaths.java | 0
.../tool/bisync/tableau/datasource/Layout.java | 0
.../bisync/tableau/datasource/SemanticValue.java | 0
.../tableau/datasource/SemanticValueList.java | 0
.../tableau/datasource/TableauConnection.java | 0
.../tableau/datasource/TableauDatasource.java | 0
.../tableau/datasource/column/Calculation.java | 0
.../bisync/tableau/datasource/column/Column.java | 0
.../bisync/tableau/datasource/connection/Col.java | 0
.../bisync/tableau/datasource/connection/Cols.java | 0
.../tableau/datasource/connection/Connection.java | 0
.../connection/ConnectionCustomization.java | 0
.../datasource/connection/NamedConnection.java | 0
.../datasource/connection/NamedConnectionList.java | 0
.../connection/customization/Customization.java | 0
.../customization/CustomizationList.java | 0
.../connection/customization/Driver.java | 0
.../connection/customization/Vendor.java | 0
.../datasource/connection/metadata/Attribute.java | 0
.../connection/metadata/AttributeList.java | 0
.../datasource/connection/metadata/Collation.java | 0
.../connection/metadata/MetadataRecord.java | 0
.../connection/metadata/MetadataRecordList.java | 0
.../datasource/connection/relation/Clause.java | 0
.../datasource/connection/relation/Expression.java | 0
.../datasource/connection/relation/Relation.java | 0
.../bisync/tableau/mapping/FunctionMapping.java | 0
.../tool/bisync/tableau/mapping/Mappings.java | 0
.../tool/bisync/tableau/mapping/TypeMapping.java | 0
.../bisync/tds/tableau.connector.template.xml | 0
.../main/resources/bisync/tds/tableau.mappings.xml | 0
.../main/resources/bisync/tds/tableau.template.xml | 0
.../kylin/rest/service/ModelTdsServiceTest.java | 6 +-
.../kylin/tool/bisync/SyncModelBuilderTest.java | 0
.../kylin/tool/bisync/SyncModelTestUtil.java | 0
.../tool/bisync/tableau/TableauDatasourceTest.java | 0
.../bisync_tableau/nmodel_basic_all_cols.tds | 0
.../bisync_tableau/nmodel_basic_inner_all_cols.tds | 0
.../nmodel_full_measure_test.connector.tds | 2 +-
.../nmodel_full_measure_test.connector_cc.tds | 0
...nmodel_full_measure_test.connector_cc_admin.tds | 0
...del_full_measure_test.connector_hierarchies.tds | 0
..._full_measure_test.connector_no_hierarchies.tds | 0
...odel_full_measure_test.connector_permission.tds | 0
...ure_test.connector_permission_agg_index_col.tds | 0
...l_measure_test.connector_permission_all_col.tds | 0
...easure_test.connector_permission_no_measure.tds | 0
...del_full_measure_test.table_index_connector.tds | 0
.../bisync_tableau/nmodel_full_measure_test.tds | 0
.../org/apache/kylin/rest/HAConfigurationTest.java | 6 +-
src/systools/pom.xml | 113 ----
src/tool/pom.xml | 10 +
.../kylin/tool/AbstractInfoExtractorTool.java | 2 +-
.../java/org/apache/kylin/tool/MetadataTool.java | 454 ++--------------
.../java/org/apache/kylin/tool/RollbackTool.java | 28 +-
.../daemon/handler/AbstractCheckStateHandler.java | 4 +-
.../apache/kylin/tool/routine/FastRoutineTool.java | 15 +-
.../org/apache/kylin/tool/routine/RoutineTool.java | 118 +----
.../kylin/tool/upgrade/UpdateUserAclTool.java | 51 +-
.../org/apache/kylin/tool/util/MetadataUtil.java | 9 +-
.../org/apache/kylin/tool/MetadataToolTest.java | 53 +-
.../tool/security/KylinPasswordResetCLITest.java | 1 +
.../kylin/tool/upgrade/UpdateUserAclToolTest.java | 5 +-
.../nmodel_full_measure_test.connector.tds | 125 -----
127 files changed, 797 insertions(+), 1528 deletions(-)
diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java
index ebf502c28b..26c02418f6 100644
--- a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java
+++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NSystemController.java
@@ -41,6 +41,7 @@ import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.AddressUtil;
+import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.cluster.ClusterManager;
@@ -58,6 +59,7 @@ import org.apache.kylin.rest.service.MetadataBackupService;
import org.apache.kylin.rest.service.ProjectService;
import org.apache.kylin.rest.service.SystemService;
import org.apache.kylin.rest.util.AclEvaluate;
+import org.apache.kylin.tool.HDFSMetadataTool;
import org.apache.kylin.tool.util.ToolUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
@@ -103,6 +105,7 @@ public class NSystemController extends NBasicController {
@Autowired
@Qualifier("projectService")
private ProjectService projectService;
+ private MetadataToolHelper metadataToolHelper = new MetadataToolHelper();
@VisibleForTesting
public void setAclEvaluate(AclEvaluate aclEvaluate) {
@@ -118,8 +121,11 @@ public class NSystemController extends NBasicController {
@GetMapping(value = "/metadata/dump")
@ResponseBody
public EnvelopeResponse<String> dumpMetadata(@RequestParam(value = "dump_path") String dumpPath) throws Exception {
- String[] args = new String[] { "-backup", "-compress", "-dir", dumpPath };
- metadataBackupService.backup(args);
+ val kylinConfig = KylinConfig.getInstanceFromEnv();
+ HDFSMetadataTool.cleanBeforeBackup(kylinConfig);
+ val backupConfig = kylinConfig.getMetadataBackupFromSystem() ? kylinConfig
+ : KylinConfig.createKylinConfig(kylinConfig);
+ metadataToolHelper.backup(backupConfig, null, dumpPath, null, true, false);
return new EnvelopeResponse<>(CODE_SUCCESS, "", "");
}
diff --git a/src/common-service/pom.xml b/src/common-service/pom.xml
index adaff3899b..db687b8f80 100644
--- a/src/common-service/pom.xml
+++ b/src/common-service/pom.xml
@@ -26,13 +26,16 @@
</parent>
<modelVersion>4.0.0</modelVersion>
<name>Kylin - Common Service</name>
- <groupId>org.apache.kylin</groupId>
<artifactId>kylin-common-service</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-tool</artifactId>
+ <artifactId>kylin-core-metadata</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-streaming</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
@@ -42,6 +45,10 @@
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-web</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-ldap</artifactId>
+ </dependency>
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/SemanticValueList.java b/src/common-service/src/main/java/org/apache/kylin/helper/HelperConstants.java
similarity index 65%
copy from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/SemanticValueList.java
copy to src/common-service/src/main/java/org/apache/kylin/helper/HelperConstants.java
index 4f89d1bf1d..9133d37013 100644
--- a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/SemanticValueList.java
+++ b/src/common-service/src/main/java/org/apache/kylin/helper/HelperConstants.java
@@ -15,17 +15,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.kylin.tool.bisync.tableau.datasource;
-import java.util.List;
+package org.apache.kylin.helper;
-import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
-import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
-public class SemanticValueList {
+/*
+ * this class is only for removing dependency of kylin-tool module, and should be refactor later
+ */
+class HelperConstants {
- @JacksonXmlProperty(localName = "semantic-value")
- @JacksonXmlElementWrapper(useWrapping = false)
- private List<SemanticValue> semanticValueList;
+ private HelperConstants() {}
+ static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss",
+ Locale.getDefault(Locale.Category.FORMAT));
}
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java b/src/common-service/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
similarity index 59%
copy from src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
copy to src/common-service/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
index daa8aed307..2804bdc4c3 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
+++ b/src/common-service/src/main/java/org/apache/kylin/helper/MetadataToolHelper.java
@@ -16,171 +16,274 @@
* limitations under the License.
*/
-package org.apache.kylin.tool;
+package org.apache.kylin.helper;
import static org.apache.kylin.common.exception.code.ErrorCodeTool.FILE_ALREADY_EXISTS;
-import static org.apache.kylin.common.exception.code.ErrorCodeTool.PARAMETER_NOT_SPECIFY;
import java.io.File;
import java.io.IOException;
import java.net.URI;
+import java.nio.file.FileSystems;
import java.nio.file.Paths;
import java.time.Clock;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
+import java.util.List;
import java.util.Locale;
import java.util.NavigableSet;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.Options;
+import javax.sql.DataSource;
+
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.ExecutableApplication;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.common.metrics.MetricsCategory;
import org.apache.kylin.common.metrics.MetricsGroup;
import org.apache.kylin.common.metrics.MetricsName;
import org.apache.kylin.common.persistence.ImageDesc;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.metadata.AuditLogStore;
+import org.apache.kylin.common.persistence.metadata.JdbcDataSource;
+import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
-import org.apache.kylin.common.util.AddressUtil;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.MetadataChecker;
-import org.apache.kylin.common.util.OptionBuilder;
-import org.apache.kylin.common.util.Unsafe;
-import org.apache.kylin.tool.util.ScreenPrintUtil;
-import org.apache.kylin.tool.util.ToolMainWrapper;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.tool.HDFSMetadataTool;
+import org.apache.kylin.tool.garbage.StorageCleaner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
import io.kyligence.kap.guava20.shaded.common.io.ByteSource;
-import lombok.Getter;
import lombok.val;
import lombok.var;
-public class MetadataTool extends ExecutableApplication {
- public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss",
- Locale.getDefault(Locale.Category.FORMAT));
- private static final Logger logger = LoggerFactory.getLogger("diag");
- private static final String HDFS_METADATA_URL_FORMATTER = "kylin_metadata@hdfs,path=%s";
+/*
+* this class is only for removing dependency of kylin-tool module, and should be refactor later
+*/
+public class MetadataToolHelper {
+ public static final DateTimeFormatter DATE_TIME_FORMATTER = HelperConstants.DATE_TIME_FORMATTER;
private static final String GLOBAL = "global";
+ private static final String HDFS_METADATA_URL_FORMATTER = "kylin_metadata@hdfs,path=%s";
- @SuppressWarnings("static-access")
- private static final Option OPERATE_BACKUP = OptionBuilder.getInstance()
- .withDescription("Backup metadata to local path or HDFS path").isRequired(false).create("backup");
-
- private static final Option OPERATE_COMPRESS = OptionBuilder.getInstance()
- .withDescription("Backup compressed metadata to HDFS path").isRequired(false).create("compress");
-
- private static final Option OPERATE_FETCH = OptionBuilder.getInstance()
- .withDescription("Fetch part of metadata to local path").isRequired(false).create("fetch");
-
- private static final Option OPERATE_LIST = OptionBuilder.getInstance()
- .withDescription("List children of target folder").isRequired(false).create("list");
-
- private static final Option OPERATE_RESTORE = OptionBuilder.getInstance()
- .withDescription("Restore metadata from local path or HDFS path").isRequired(false).create("restore");
-
- private static final Option OPTION_AFTER_TRUNCATE = OptionBuilder.getInstance()
- .withDescription("Restore overwrite metadata from local path or HDFS path (optional)").isRequired(false)
- .withLongOpt("after-truncate").hasArg(false).create("d");
-
- private static final Option OPTION_DIR = OptionBuilder.getInstance().hasArg().withArgName("DIRECTORY_PATH")
- .withDescription("Specify the target directory for backup and restore").isRequired(false).create("dir");
+ private static final Logger logger = LoggerFactory.getLogger(MetadataToolHelper.class);
- private static final Option OPTION_PROJECT = OptionBuilder.getInstance().hasArg().withArgName("PROJECT_NAME")
- .withDescription("Specify project level backup and restore (optional)").isRequired(false).create("project");
+ public void rotateAuditLog() {
+ val resourceStore = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
+ val auditLogStore = resourceStore.getAuditLogStore();
+ auditLogStore.rotate();
+ }
- private static final Option OPTION_TARGET = OptionBuilder.getInstance().hasArg().withArgName("TARGET_FILE")
- .withDescription("Specify part of metadata for fetch to local path").isRequired(false).create("target");
+ public void backup(KylinConfig kylinConfig) throws Exception {
+ HDFSMetadataTool.cleanBeforeBackup(kylinConfig);
+ new MetadataToolHelper().backup(kylinConfig, null, HadoopUtil.getBackupFolder(kylinConfig), null, true, false);
+ }
- private static final Option FOLDER_NAME = OptionBuilder.getInstance().hasArg().withArgName("FOLDER_NAME")
- .withDescription("Specify the folder name for backup").isRequired(false).create("folder");
+ public void backup(KylinConfig kylinConfig, String dir, String folder) throws Exception {
+ HDFSMetadataTool.cleanBeforeBackup(kylinConfig);
+ new MetadataToolHelper().backup(kylinConfig, null, dir, folder, true, false);
+ }
- private static final Option OPTION_EXCLUDE_TABLE_EXD = OptionBuilder.getInstance()
- .withDescription("Exclude metadata {project}/table_exd directory").isRequired(false)
- .create("excludeTableExd");
+ public void backup(KylinConfig kylinConfig, String project, String path, String folder, boolean compress,
+ boolean excludeTableExd) throws Exception {
+ boolean isGlobal = null == project;
+ long startAt = System.currentTimeMillis();
+ try {
+ doBackup(kylinConfig, project, path, folder, compress, excludeTableExd);
+ } catch (Exception be) {
+ if (isGlobal) {
+ MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_FAILED, MetricsCategory.GLOBAL, GLOBAL);
+ } else {
+ MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_FAILED, MetricsCategory.PROJECT, project);
+ }
+ throw be;
+ } finally {
+ if (isGlobal) {
+ MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP, MetricsCategory.GLOBAL, GLOBAL);
+ MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_DURATION, MetricsCategory.GLOBAL, GLOBAL,
+ System.currentTimeMillis() - startAt);
+ } else {
+ MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP, MetricsCategory.PROJECT, project);
+ MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_DURATION, MetricsCategory.PROJECT, project,
+ System.currentTimeMillis() - startAt);
+ }
+ }
+ }
- private final Options options;
+ void doBackup(KylinConfig kylinConfig, String project, String path, String folder, boolean compress,
+ boolean excludeTableExd) throws Exception {
+ ResourceStore resourceStore = ResourceStore.getKylinMetaStore(kylinConfig);
+ boolean isUTEnv = kylinConfig.isUTEnv();
- private final KylinConfig kylinConfig;
+ if (StringUtils.isBlank(path)) {
+ path = KylinConfigBase.getKylinHome() + File.separator + "meta_backups";
+ }
+ if (StringUtils.isEmpty(folder)) {
+ folder = LocalDateTime.now(Clock.systemDefaultZone()).format(MetadataToolHelper.DATE_TIME_FORMATTER)
+ + "_backup";
+ }
+ String backupPath = StringUtils.appendIfMissing(path, "/") + folder;
+ logger.info("The metadata backup path is {}}", backupPath);
+ val backupMetadataUrl = getMetadataUrl(backupPath, compress, kylinConfig);
+ val backupConfig = KylinConfig.createKylinConfig(kylinConfig);
+ backupConfig.setMetadataUrl(backupMetadataUrl);
+ abortIfAlreadyExists(backupPath);
+ logger.info("The backup metadataUrl is {} and backup path is {}", backupMetadataUrl, backupPath);
+ try (val backupResourceStore = ResourceStore.getKylinMetaStore(backupConfig)) {
+ val backupMetadataStore = backupResourceStore.getMetadataStore();
+ if (StringUtils.isBlank(project)) {
+ logger.info("start to copy all projects from ResourceStore.");
+ long finalOffset = getOffset(isUTEnv, resourceStore);
+ backupResourceStore.putResourceWithoutCheck(ResourceStore.METASTORE_IMAGE,
+ ByteSource.wrap(JsonUtil.writeValueAsBytes(new ImageDesc(finalOffset))),
+ System.currentTimeMillis(), -1);
+ var projectFolders = resourceStore.listResources("/");
+ if (projectFolders == null) {
+ return;
+ }
+ UnitOfWork.doInTransactionWithRetry(() -> {
+ backupProjects(projectFolders, resourceStore, backupResourceStore, excludeTableExd);
+ return null;
+ }, UnitOfWork.GLOBAL_UNIT);
- private ResourceStore resourceStore;
+ val uuid = resourceStore.getResource(ResourceStore.METASTORE_UUID_TAG);
+ if (uuid != null) {
+ backupResourceStore.putResourceWithoutCheck(uuid.getResPath(), uuid.getByteSource(),
+ uuid.getTimestamp(), -1);
+ }
+ logger.info("start to backup all projects");
- @Getter
- private String backupPath;
+ } else {
+ logger.info("start to copy project {} from ResourceStore.", project);
+ UnitOfWork.doInTransactionWithRetry(
+ UnitOfWorkParams.builder().readonly(true).unitName(project).processor(() -> {
+ copyResourceStore("/" + project, resourceStore, backupResourceStore, true, excludeTableExd);
+ val uuid = resourceStore.getResource(ResourceStore.METASTORE_UUID_TAG);
+ backupResourceStore.putResourceWithoutCheck(uuid.getResPath(), uuid.getByteSource(),
+ uuid.getTimestamp(), -1);
+ return null;
+ }).build());
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException("metadata task is interrupt");
+ }
+ logger.info("start to backup project {}", project);
+ }
+ backupResourceStore.deleteResource(ResourceStore.METASTORE_TRASH_RECORD);
+ backupMetadataStore.dump(backupResourceStore);
+ logger.info("backup successfully at {}", backupPath);
+ }
+ }
- @Getter
- private String fetchPath;
+ public String getMetadataUrl(String rootPath, boolean compressed, KylinConfig kylinConfig) {
+ if (HadoopUtil.isHdfsCompatibleSchema(rootPath, kylinConfig)) {
+ val url = String.format(Locale.ROOT, HDFS_METADATA_URL_FORMATTER,
+ Path.getPathWithoutSchemeAndAuthority(new Path(rootPath)).toString() + "/");
+ return compressed ? url + ",zip=1" : url;
+ } else if (rootPath.startsWith("file://")) {
+ rootPath = rootPath.replace("file://", "");
+ return StringUtils.appendIfMissing(rootPath, "/");
- MetadataTool() {
- kylinConfig = KylinConfig.getInstanceFromEnv();
- this.options = new Options();
- initOptions();
+ } else {
+ return StringUtils.appendIfMissing(rootPath, "/");
+ }
}
- public MetadataTool(KylinConfig kylinConfig) {
- this.kylinConfig = kylinConfig;
- this.options = new Options();
- initOptions();
+ private void backupProjects(NavigableSet<String> projectFolders, ResourceStore resourceStore,
+ ResourceStore backupResourceStore, boolean excludeTableExd) throws InterruptedException {
+ for (String projectPath : projectFolders) {
+ if (projectPath.equals(ResourceStore.METASTORE_UUID_TAG)
+ || projectPath.equals(ResourceStore.METASTORE_IMAGE)) {
+ continue;
+ }
+ // The "_global" directory is already included in the full backup
+ copyResourceStore(projectPath, resourceStore, backupResourceStore, false, excludeTableExd);
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException("metadata task is interrupt");
+ }
+ }
}
- public static void backup(KylinConfig kylinConfig) throws IOException {
- HDFSMetadataTool.cleanBeforeBackup(kylinConfig);
- String[] args = new String[] { "-backup", "-compress", "-dir", HadoopUtil.getBackupFolder(kylinConfig) };
- val backupTool = new MetadataTool(kylinConfig);
- backupTool.execute(args);
+ private void copyResourceStore(String projectPath, ResourceStore srcResourceStore,
+ ResourceStore destResourceStore, boolean isProjectLevel, boolean excludeTableExd) {
+ if (excludeTableExd) {
+ String tableExdPath = projectPath + ResourceStore.TABLE_EXD_RESOURCE_ROOT;
+ var projectItems = srcResourceStore.listResources(projectPath);
+ for (String item : projectItems) {
+ if (item.equals(tableExdPath)) {
+ continue;
+ }
+ srcResourceStore.copy(item, destResourceStore);
+ }
+ } else {
+ srcResourceStore.copy(projectPath, destResourceStore);
+ }
+ if (isProjectLevel) {
+ // The project-level backup needs to contain "/_global/project/*.json"
+ val projectName = Paths.get(projectPath).getFileName().toString();
+ srcResourceStore.copy(ProjectInstance.concatResourcePath(projectName), destResourceStore);
+ }
}
- public static void backup(KylinConfig kylinConfig, String dir, String folder) throws IOException {
- HDFSMetadataTool.cleanBeforeBackup(kylinConfig);
- String[] args = new String[] { "-backup", "-compress", "-dir", dir, "-folder", folder };
- val backupTool = new MetadataTool(kylinConfig);
- backupTool.execute(args);
+ private long getOffset(boolean isUTEnv, ResourceStore resourceStore) {
+ AuditLogStore auditLogStore = resourceStore.getAuditLogStore();
+ if (isUTEnv) {
+ return auditLogStore.getMaxId();
+ } else {
+ return auditLogStore.getLogOffset() == 0 ? resourceStore.getOffset() : auditLogStore.getLogOffset();
+ }
}
- public static void restore(KylinConfig kylinConfig, String folder) throws IOException {
- val tool = new MetadataTool(kylinConfig);
- tool.execute(new String[] { "-restore", "-dir", folder, "--after-truncate" });
+ private void abortIfAlreadyExists(String path) throws IOException {
+ URI uri = HadoopUtil.makeURI(path);
+ if (!uri.isAbsolute()) {
+ logger.info("no scheme specified for {}, try local file system file://", path);
+ File localFile = new File(path);
+ if (localFile.exists()) {
+ logger.error("[UNEXPECTED_THINGS_HAPPENED] local file {} already exists ", path);
+ throw new KylinException(FILE_ALREADY_EXISTS, path);
+ }
+ return;
+ }
+ val fs = HadoopUtil.getWorkingFileSystem();
+ if (fs.exists(new Path(path))) {
+ logger.error("[UNEXPECTED_THINGS_HAPPENED] specified file {} already exists ", path);
+ throw new KylinException(FILE_ALREADY_EXISTS, path);
+ }
}
- public static void main(String[] args) {
- ToolMainWrapper.wrap(args, () -> {
- val config = KylinConfig.getInstanceFromEnv();
- val tool = new MetadataTool(config);
- val optionsHelper = new OptionsHelper();
- optionsHelper.parseOptions(tool.getOptions(), args);
- boolean isBackup = optionsHelper.hasOption(OPERATE_BACKUP);
- boolean isFetch = optionsHelper.hasOption(OPERATE_FETCH);
- if ((isBackup || isFetch) && ScreenPrintUtil.isMainThread()) {
- config.setProperty("kylin.env.metadata.only-for-read", "true");
- }
- val resourceStore = ResourceStore.getKylinMetaStore(config);
- resourceStore.getAuditLogStore().setInstance(AddressUtil.getMockPortAddress());
- tool.execute(args);
- if (isBackup && StringUtils.isNotEmpty(tool.getBackupPath())) {
- System.out.printf(Locale.ROOT, "The metadata backup path is %s.%n", tool.getBackupPath());
- }
- });
- Unsafe.systemExit(0);
+ public void restore(KylinConfig kylinConfig, String project, String path, boolean delete) throws Exception {
+ logger.info("Restore metadata with delete : {}", delete);
+ ResourceStore resourceStore = ResourceStore.getKylinMetaStore(kylinConfig);
+ val restoreMetadataUrl = getMetadataUrl(path, false, kylinConfig);
+ val restoreConfig = KylinConfig.createKylinConfig(kylinConfig);
+ restoreConfig.setMetadataUrl(restoreMetadataUrl);
+ logger.info("The restore metadataUrl is {} and restore path is {} ", restoreMetadataUrl, path);
+
+ val restoreResourceStore = ResourceStore.getKylinMetaStore(restoreConfig);
+ val restoreMetadataStore = restoreResourceStore.getMetadataStore();
+ MetadataChecker metadataChecker = new MetadataChecker(restoreMetadataStore);
+
+ val verifyResult = metadataChecker.verify();
+ Preconditions.checkState(verifyResult.isQualified(),
+ verifyResult.getResultMessage() + "\n the metadata dir is not qualified");
+ restore(resourceStore, restoreResourceStore, project, delete);
+ backup(kylinConfig);
+
}
- public static void restore(ResourceStore currentResourceStore, ResourceStore restoreResourceStore, String project,
+ public void restore(ResourceStore currentResourceStore, ResourceStore restoreResourceStore, String project,
boolean delete) {
if (StringUtils.isBlank(project)) {
logger.info("start to restore all projects");
@@ -222,7 +325,7 @@ public class MetadataTool extends ExecutableApplication {
UnitOfWork.doInTransactionWithRetry(() -> doRestore(currentResourceStore, restoreResourceStore,
finalGlobalDestResources, globalSrcResources, delete), UnitOfWork.GLOBAL_UNIT, 1);
- val projectPath = "/" + project;
+ val projectPath = FileSystems.getDefault().getSeparator() + project;
val destResources = currentResourceStore.listResourcesRecursively(projectPath);
val srcResources = restoreResourceStore.listResourcesRecursively(projectPath);
@@ -234,7 +337,7 @@ public class MetadataTool extends ExecutableApplication {
logger.info("restore successfully");
}
- private static int doRestore(ResourceStore currentResourceStore, ResourceStore restoreResourceStore,
+ private int doRestore(ResourceStore currentResourceStore, ResourceStore restoreResourceStore,
Set<String> destResources, Set<String> srcResources, boolean delete) throws IOException {
val threadViewRS = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
@@ -267,94 +370,29 @@ public class MetadataTool extends ExecutableApplication {
return 0;
}
- private void initOptions() {
- final OptionGroup optionGroup = new OptionGroup();
- optionGroup.setRequired(true);
- optionGroup.addOption(OPERATE_BACKUP);
- optionGroup.addOption(OPERATE_FETCH);
- optionGroup.addOption(OPERATE_LIST);
- optionGroup.addOption(OPERATE_RESTORE);
-
- options.addOptionGroup(optionGroup);
- options.addOption(OPTION_DIR);
- options.addOption(OPTION_PROJECT);
- options.addOption(FOLDER_NAME);
- options.addOption(OPTION_TARGET);
- options.addOption(OPERATE_COMPRESS);
- options.addOption(OPTION_EXCLUDE_TABLE_EXD);
- options.addOption(OPTION_AFTER_TRUNCATE);
- }
-
- @Override
- protected Options getOptions() {
- return options;
- }
-
- @Override
- protected void execute(OptionsHelper optionsHelper) throws Exception {
- logger.info("start to init ResourceStore");
- resourceStore = ResourceStore.getKylinMetaStore(kylinConfig);
- if (optionsHelper.hasOption(OPERATE_BACKUP)) {
- boolean isGlobal = null == optionsHelper.getOptionValue(OPTION_PROJECT);
- long startAt = System.currentTimeMillis();
-
- try {
- backup(optionsHelper);
- } catch (Exception be) {
- if (isGlobal) {
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_FAILED, MetricsCategory.GLOBAL, GLOBAL);
- } else {
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_FAILED, MetricsCategory.PROJECT,
- optionsHelper.getOptionValue(OPTION_PROJECT));
- }
- throw be;
- } finally {
- if (isGlobal) {
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP, MetricsCategory.GLOBAL, GLOBAL);
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_DURATION, MetricsCategory.GLOBAL, GLOBAL,
- System.currentTimeMillis() - startAt);
- } else {
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP, MetricsCategory.PROJECT,
- optionsHelper.getOptionValue(OPTION_PROJECT));
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_DURATION, MetricsCategory.PROJECT,
- optionsHelper.getOptionValue(OPTION_PROJECT), System.currentTimeMillis() - startAt);
- }
- }
-
- } else if (optionsHelper.hasOption(OPERATE_FETCH)) {
- fetch(optionsHelper);
- } else if (optionsHelper.hasOption(OPERATE_LIST)) {
- list(optionsHelper);
- } else if (optionsHelper.hasOption(OPERATE_RESTORE)) {
- restore(optionsHelper, optionsHelper.hasOption(OPTION_AFTER_TRUNCATE));
- } else {
- throw new KylinException(PARAMETER_NOT_SPECIFY, "-restore");
+ public void cleanStorage(boolean storageCleanup, List<String> projects, double requestFSRate,
+ int retryTimes) {
+ try {
+ StorageCleaner storageCleaner = new StorageCleaner(storageCleanup, projects, requestFSRate, retryTimes);
+ System.out.println("Start to cleanup HDFS");
+ storageCleaner.execute();
+ System.out.println("cleanup HDFS finished");
+ } catch (Exception e) {
+ logger.error("cleanup HDFS failed", e);
+ System.out.println(StorageCleaner.ANSI_RED
+ + "cleanup HDFS failed. Detailed Message is at ${KYLIN_HOME}/logs/shell.stderr"
+ + StorageCleaner.ANSI_RESET);
}
}
- private void abortIfAlreadyExists(String path) throws IOException {
- URI uri = HadoopUtil.makeURI(path);
- if (!uri.isAbsolute()) {
- logger.info("no scheme specified for {}, try local file system file://", path);
- File localFile = new File(path);
- if (localFile.exists()) {
- logger.error("[UNEXPECTED_THINGS_HAPPENED] local file {} already exists ", path);
- throw new KylinException(FILE_ALREADY_EXISTS, path);
- }
- return;
- }
- val fs = HadoopUtil.getWorkingFileSystem();
- if (fs.exists(new Path(path))) {
- logger.error("[UNEXPECTED_THINGS_HAPPENED] specified file {} already exists ", path);
- throw new KylinException(FILE_ALREADY_EXISTS, path);
- }
+ public DataSource getDataSource(KylinConfig kylinConfig) throws Exception {
+ val url = kylinConfig.getMetadataUrl();
+ val props = JdbcUtil.datasourceParameters(url);
+ return JdbcDataSource.getDataSource(props);
}
- private void fetch(OptionsHelper optionsHelper) throws Exception {
- var path = optionsHelper.getOptionValue(OPTION_DIR);
- var folder = optionsHelper.getOptionValue(FOLDER_NAME);
- val excludeTableExd = optionsHelper.hasOption(OPTION_EXCLUDE_TABLE_EXD);
- val target = optionsHelper.getOptionValue(OPTION_TARGET);
+ public void fetch(KylinConfig kylinConfig, String path, String folder, String target, boolean excludeTableExd) throws Exception {
+ ResourceStore resourceStore = ResourceStore.getKylinMetaStore(kylinConfig);
if (StringUtils.isBlank(path)) {
path = KylinConfigBase.getKylinHome() + File.separator + "meta_fetch";
}
@@ -364,9 +402,9 @@ public class MetadataTool extends ExecutableApplication {
if (target == null) {
System.out.println("target file must be set with fetch mode");
} else {
- fetchPath = StringUtils.appendIfMissing(path, "/") + folder;
+ val fetchPath = StringUtils.appendIfMissing(path, "/") + folder;
// currently do not support compress with fetch
- val fetchMetadataUrl = getMetadataUrl(fetchPath, false);
+ val fetchMetadataUrl = getMetadataUrl(fetchPath, false, kylinConfig);
val fetchConfig = KylinConfig.createKylinConfig(kylinConfig);
fetchConfig.setMetadataUrl(fetchMetadataUrl);
abortIfAlreadyExists(fetchPath);
@@ -401,8 +439,8 @@ public class MetadataTool extends ExecutableApplication {
}
}
- private NavigableSet<String> list(OptionsHelper optionsHelper) throws Exception {
- val target = optionsHelper.getOptionValue(OPTION_TARGET);
+ public NavigableSet<String> list(KylinConfig kylinConfig, String target) throws Exception {
+ ResourceStore resourceStore = ResourceStore.getKylinMetaStore(kylinConfig);
var res = resourceStore.listResources(target);
if (res == null) {
System.out.printf("%s is not exist%n", target);
@@ -412,154 +450,4 @@ public class MetadataTool extends ExecutableApplication {
return res;
}
- private void backup(OptionsHelper optionsHelper) throws Exception {
- val project = optionsHelper.getOptionValue(OPTION_PROJECT);
- var path = optionsHelper.getOptionValue(OPTION_DIR);
- var folder = optionsHelper.getOptionValue(FOLDER_NAME);
- var compress = optionsHelper.hasOption(OPERATE_COMPRESS);
- val excludeTableExd = optionsHelper.hasOption(OPTION_EXCLUDE_TABLE_EXD);
- if (StringUtils.isBlank(path)) {
- path = KylinConfigBase.getKylinHome() + File.separator + "meta_backups";
- }
- if (StringUtils.isEmpty(folder)) {
- folder = LocalDateTime.now(Clock.systemDefaultZone()).format(DATE_TIME_FORMATTER) + "_backup";
- }
- backupPath = StringUtils.appendIfMissing(path, "/") + folder;
- val backupMetadataUrl = getMetadataUrl(backupPath, compress);
- val backupConfig = KylinConfig.createKylinConfig(kylinConfig);
- backupConfig.setMetadataUrl(backupMetadataUrl);
- abortIfAlreadyExists(backupPath);
- logger.info("The backup metadataUrl is {} and backup path is {}", backupMetadataUrl, backupPath);
-
- try (val backupResourceStore = ResourceStore.getKylinMetaStore(backupConfig)) {
-
- val backupMetadataStore = backupResourceStore.getMetadataStore();
-
- if (StringUtils.isBlank(project)) {
- logger.info("start to copy all projects from ResourceStore.");
- val auditLogStore = resourceStore.getAuditLogStore();
- long finalOffset = getOffset(auditLogStore);
- backupResourceStore.putResourceWithoutCheck(ResourceStore.METASTORE_IMAGE,
- ByteSource.wrap(JsonUtil.writeValueAsBytes(new ImageDesc(finalOffset))),
- System.currentTimeMillis(), -1);
- var projectFolders = resourceStore.listResources("/");
- if (projectFolders == null) {
- return;
- }
- UnitOfWork.doInTransactionWithRetry(() -> {
- backupProjects(projectFolders, backupResourceStore, excludeTableExd);
- return null;
- }, UnitOfWork.GLOBAL_UNIT);
-
- val uuid = resourceStore.getResource(ResourceStore.METASTORE_UUID_TAG);
- if (uuid != null) {
- backupResourceStore.putResourceWithoutCheck(uuid.getResPath(), uuid.getByteSource(),
- uuid.getTimestamp(), -1);
- }
- logger.info("start to backup all projects");
-
- } else {
- logger.info("start to copy project {} from ResourceStore.", project);
- UnitOfWork.doInTransactionWithRetry(
- UnitOfWorkParams.builder().readonly(true).unitName(project).processor(() -> {
- copyResourceStore("/" + project, resourceStore, backupResourceStore, true, excludeTableExd);
- val uuid = resourceStore.getResource(ResourceStore.METASTORE_UUID_TAG);
- backupResourceStore.putResourceWithoutCheck(uuid.getResPath(), uuid.getByteSource(),
- uuid.getTimestamp(), -1);
- return null;
- }).build());
- if (Thread.currentThread().isInterrupted()) {
- throw new InterruptedException("metadata task is interrupt");
- }
- logger.info("start to backup project {}", project);
- }
- backupResourceStore.deleteResource(ResourceStore.METASTORE_TRASH_RECORD);
- backupMetadataStore.dump(backupResourceStore);
- logger.info("backup successfully at {}", backupPath);
- }
- }
-
- private long getOffset(AuditLogStore auditLogStore) {
- long offset = 0;
- if (kylinConfig.isUTEnv())
- offset = auditLogStore.getMaxId();
- else
- offset = auditLogStore.getLogOffset() == 0 ? resourceStore.getOffset() : auditLogStore.getLogOffset();
- return offset;
- }
-
- private void backupProjects(NavigableSet<String> projectFolders, ResourceStore backupResourceStore,
- boolean excludeTableExd) throws InterruptedException {
- for (String projectPath : projectFolders) {
- if (projectPath.equals(ResourceStore.METASTORE_UUID_TAG)
- || projectPath.equals(ResourceStore.METASTORE_IMAGE)) {
- continue;
- }
- // The "_global" directory is already included in the full backup
- copyResourceStore(projectPath, resourceStore, backupResourceStore, false, excludeTableExd);
- if (Thread.currentThread().isInterrupted()) {
- throw new InterruptedException("metadata task is interrupt");
- }
- }
- }
-
- private void copyResourceStore(String projectPath, ResourceStore srcResourceStore, ResourceStore destResourceStore,
- boolean isProjectLevel, boolean excludeTableExd) {
- if (excludeTableExd) {
- String tableExdPath = projectPath + ResourceStore.TABLE_EXD_RESOURCE_ROOT;
- var projectItems = srcResourceStore.listResources(projectPath);
- for (String item : projectItems) {
- if (item.equals(tableExdPath)) {
- continue;
- }
- srcResourceStore.copy(item, destResourceStore);
- }
- } else {
- srcResourceStore.copy(projectPath, destResourceStore);
- }
- if (isProjectLevel) {
- // The project-level backup needs to contain "/_global/project/*.json"
- val projectName = Paths.get(projectPath).getFileName().toString();
- srcResourceStore.copy(ProjectInstance.concatResourcePath(projectName), destResourceStore);
- }
- }
-
- private void restore(OptionsHelper optionsHelper, boolean delete) throws IOException {
- logger.info("Restore metadata with delete : {}", delete);
- val project = optionsHelper.getOptionValue(OPTION_PROJECT);
- val restorePath = optionsHelper.getOptionValue(OPTION_DIR);
-
- val restoreMetadataUrl = getMetadataUrl(restorePath, false);
- val restoreConfig = KylinConfig.createKylinConfig(kylinConfig);
- restoreConfig.setMetadataUrl(restoreMetadataUrl);
- logger.info("The restore metadataUrl is {} and restore path is {} ", restoreMetadataUrl, restorePath);
-
- val restoreResourceStore = ResourceStore.getKylinMetaStore(restoreConfig);
- val restoreMetadataStore = restoreResourceStore.getMetadataStore();
- MetadataChecker metadataChecker = new MetadataChecker(restoreMetadataStore);
-
- val verifyResult = metadataChecker.verify();
- if (!verifyResult.isQualified()) {
- throw new RuntimeException(verifyResult.getResultMessage() + "\n the metadata dir is not qualified");
- }
- restore(resourceStore, restoreResourceStore, project, delete);
- backup(kylinConfig);
-
- }
-
- String getMetadataUrl(String rootPath, boolean compressed) {
- if (HadoopUtil.isHdfsCompatibleSchema(rootPath, kylinConfig)) {
- val url = String.format(Locale.ROOT, HDFS_METADATA_URL_FORMATTER,
- Path.getPathWithoutSchemeAndAuthority(new Path(rootPath)).toString() + "/");
- return compressed ? url + ",zip=1" : url;
-
- } else if (rootPath.startsWith("file://")) {
- rootPath = rootPath.replace("file://", "");
- return StringUtils.appendIfMissing(rootPath, "/");
-
- } else {
- return StringUtils.appendIfMissing(rootPath, "/");
-
- }
- }
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/helper/RoutineToolHelper.java b/src/common-service/src/main/java/org/apache/kylin/helper/RoutineToolHelper.java
new file mode 100644
index 0000000000..8c9b06d441
--- /dev/null
+++ b/src/common-service/src/main/java/org/apache/kylin/helper/RoutineToolHelper.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.helper;
+
+import lombok.extern.slf4j.Slf4j;
+import lombok.val;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.common.util.SetThreadName;
+import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
+import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.query.util.QueryHisStoreUtil;
+import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
+import org.apache.kylin.metadata.streaming.util.StreamingJobRecordStoreUtil;
+import org.apache.kylin.metadata.streaming.util.StreamingJobStatsStoreUtil;
+import org.apache.kylin.tool.garbage.GarbageCleaner;
+import org.apache.kylin.tool.garbage.SourceUsageCleaner;
+import org.apache.kylin.tool.garbage.StorageCleaner;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+/*
+ * this class is only for removing dependency of kylin-tool module, and should be refactor later
+ */
+@Slf4j
+public class RoutineToolHelper {
+
+ private RoutineToolHelper() {
+ }
+
+ public static void cleanQueryHistories() {
+ QueryHisStoreUtil.cleanQueryHistory();
+ }
+
+ public static void cleanStreamingStats() {
+ StreamingJobStatsStoreUtil.cleanStreamingJobStats();
+ StreamingJobRecordStoreUtil.cleanStreamingJobRecord();
+ }
+
+ public static void deleteRawRecItems() {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ List<ProjectInstance> projectInstances = NProjectManager.getInstance(config).listAllProjects().stream()
+ .filter(projectInstance -> !projectInstance.isExpertMode()).collect(Collectors.toList());
+ if (projectInstances.isEmpty()) {
+ return;
+ }
+ try (SetThreadName ignored = new SetThreadName("DeleteRawRecItemsInDB")) {
+ val jdbcRawRecStore = new JdbcRawRecStore(KylinConfig.getInstanceFromEnv());
+ jdbcRawRecStore.deleteOutdated();
+ } catch (Exception e) {
+ log.error("delete outdated advice fail: ", e);
+ }
+ }
+
+ public static void cleanGlobalSourceUsage() {
+ log.info("Start to clean up global meta");
+ try {
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ new SourceUsageCleaner().cleanup();
+ return null;
+ }, UnitOfWork.GLOBAL_UNIT);
+ } catch (Exception e) {
+ log.error("Failed to clean global meta", e);
+ }
+ log.info("Clean up global meta finished");
+
+ }
+
+ public static void cleanMetaByProject(String projectName) {
+ log.info("Start to clean up {} meta", projectName);
+ try {
+ GarbageCleaner.cleanMetadata(projectName);
+ } catch (Exception e) {
+ log.error("Project[{}] cleanup Metadata failed", projectName, e);
+ }
+ log.info("Clean up {} meta finished", projectName);
+ }
+
+ public static void cleanMeta(List<String> projectsToCleanup) {
+ try {
+ cleanGlobalSourceUsage();
+ for (String projName : projectsToCleanup) {
+ cleanMetaByProject(projName);
+ }
+ cleanQueryHistories();
+ cleanStreamingStats();
+ deleteRawRecItems();
+ System.out.println("Metadata cleanup finished");
+ } catch (Exception e) {
+ log.error("Metadata cleanup failed", e);
+ System.out.println(StorageCleaner.ANSI_RED
+ + "Metadata cleanup failed. Detailed Message is at ${KYLIN_HOME}/logs/shell.stderr"
+ + StorageCleaner.ANSI_RESET);
+ }
+
+ }
+
+}
diff --git a/src/common-service/src/main/java/org/apache/kylin/helper/UpdateUserAclToolHelper.java b/src/common-service/src/main/java/org/apache/kylin/helper/UpdateUserAclToolHelper.java
new file mode 100644
index 0000000000..a73fd07ce8
--- /dev/null
+++ b/src/common-service/src/main/java/org/apache/kylin/helper/UpdateUserAclToolHelper.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.helper;
+
+import lombok.val;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.EncryptUtil;
+import org.apache.kylin.metadata.upgrade.GlobalAclVersionManager;
+import org.apache.kylin.tool.util.LdapUtils;
+import org.springframework.security.ldap.DefaultSpringSecurityContextSource;
+import org.springframework.security.ldap.SpringSecurityLdapTemplate;
+
+import javax.naming.directory.SearchControls;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Set;
+
+public class UpdateUserAclToolHelper {
+ private UpdateUserAclToolHelper() {}
+
+ public static UpdateUserAclToolHelper getInstance() {
+ return new UpdateUserAclToolHelper();
+ }
+
+ public Set<String> getLdapAdminUsers() {
+ val ldapTemplate = createLdapTemplate();
+ val ldapUserDNs = LdapUtils.getAllGroupMembers(ldapTemplate,
+ KylinConfig.getInstanceFromEnv().getLDAPAdminRole());
+ val searchControls = new SearchControls();
+ searchControls.setSearchScope(2);
+ Map<String, String> dnMapperMap = LdapUtils.getAllValidUserDnMap(ldapTemplate, searchControls);
+ val users = new HashSet<String>();
+ for (String u : ldapUserDNs) {
+ Optional.ofNullable(dnMapperMap.get(u)).ifPresent(users::add);
+ }
+ return users;
+ }
+
+ private SpringSecurityLdapTemplate createLdapTemplate() {
+ val properties = KylinConfig.getInstanceFromEnv().exportToProperties();
+ val contextSource = new DefaultSpringSecurityContextSource(
+ properties.getProperty("kylin.security.ldap.connection-server"));
+ contextSource.setUserDn(properties.getProperty("kylin.security.ldap.connection-username"));
+ contextSource.setPassword(getPassword(properties));
+ contextSource.afterPropertiesSet();
+ return new SpringSecurityLdapTemplate(contextSource);
+ }
+
+ public String getPassword(Properties properties) {
+ val password = properties.getProperty("kylin.security.ldap.connection-password");
+ return EncryptUtil.decrypt(password);
+ }
+
+ public boolean isUpgraded() {
+ val versionManager = GlobalAclVersionManager.getInstance(KylinConfig.getInstanceFromEnv());
+ return versionManager.exists();
+ }
+
+
+
+
+}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/security/AdminUserAspect.java b/src/common-service/src/main/java/org/apache/kylin/rest/security/AdminUserAspect.java
index b9f4af991d..f31598aed4 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/security/AdminUserAspect.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/security/AdminUserAspect.java
@@ -24,8 +24,8 @@ import java.util.Objects;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.scheduler.EventBusFactory;
+import org.apache.kylin.metadata.upgrade.GlobalAclVersionManager;
import org.apache.kylin.rest.service.UserAclService;
-import org.apache.kylin.tool.upgrade.UpdateUserAclTool;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Aspect;
import org.springframework.beans.factory.annotation.Autowired;
@@ -41,14 +41,22 @@ import lombok.extern.slf4j.Slf4j;
public class AdminUserAspect {
private List<String> adminUserList;
- private UpdateUserAclTool tool = new UpdateUserAclTool();
-
@Autowired
@Qualifier("userAclService")
private UserAclService userAclService;
private boolean superAdminInitialized = false;
+ private boolean isUpgraded() {
+ val versionManager = GlobalAclVersionManager.getInstance(KylinConfig.getInstanceFromEnv());
+ return versionManager.exists();
+ }
+
+ private boolean isAdminUserUpgraded() {
+ val userAclManager = UserAclManager.getInstance(KylinConfig.getInstanceFromEnv());
+ return userAclManager.listAclUsernames().size() > 0;
+ }
+
@AfterReturning(value = "execution(* org.apache.kylin.rest.service.OpenUserService.listAdminUsers(..))", returning = "adminUserList")
public void doAfterListAdminUsers(List<String> adminUserList) {
val kylinConfig = KylinConfig.getInstanceFromEnv();
@@ -56,7 +64,7 @@ public class AdminUserAspect {
return;
}
// upgrade admin user acl from job node
- if (kylinConfig.isJobNode() && tool.isUpgraded() && !tool.isAdminUserUpgraded()) {
+ if (kylinConfig.isJobNode() && isUpgraded() && !isAdminUserUpgraded()) {
userAclService.syncAdminUserAcl(adminUserList, false);
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/MetadataBackupService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/MetadataBackupService.java
index 63c4d22329..bb9fb9bd34 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/MetadataBackupService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/MetadataBackupService.java
@@ -17,17 +17,15 @@
*/
package org.apache.kylin.rest.service;
-import java.io.IOException;
import java.time.Clock;
import java.time.LocalDateTime;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.SetThreadName;
+import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.tool.HDFSMetadataTool;
-import org.apache.kylin.tool.MetadataTool;
import org.springframework.stereotype.Service;
import lombok.SneakyThrows;
@@ -36,42 +34,35 @@ import lombok.val;
@Service
public class MetadataBackupService {
- @SneakyThrows(IOException.class)
- public void backupAll(){
+ private final MetadataToolHelper helper = new MetadataToolHelper();
+
+ @SneakyThrows(Exception.class)
+ public void backupAll() {
try (SetThreadName ignored = new SetThreadName("MetadataBackupWorker")) {
- String[] args = new String[] { "-backup", "-compress", "-dir", getBackupDir() };
- backup(args);
- rotateAuditLog();
+ val kylinConfig = KylinConfig.getInstanceFromEnv();
+ HDFSMetadataTool.cleanBeforeBackup(kylinConfig);
+ val backupConfig = kylinConfig.getMetadataBackupFromSystem() ? kylinConfig
+ : KylinConfig.createKylinConfig(kylinConfig);
+ helper.backup(backupConfig, null, getBackupDir(kylinConfig), null, true, false);
+ helper.rotateAuditLog();
}
}
- public void backup(String[] args) throws IOException {
+ public String backupProject(String project) throws Exception {
+ val folder = LocalDateTime.now(Clock.systemDefaultZone()).format(MetadataToolHelper.DATE_TIME_FORMATTER)
+ + "_backup";
val kylinConfig = KylinConfig.getInstanceFromEnv();
- HDFSMetadataTool.cleanBeforeBackup(KylinConfig.getInstanceFromEnv());
+ HDFSMetadataTool.cleanBeforeBackup(kylinConfig);
val backupConfig = kylinConfig.getMetadataBackupFromSystem() ? kylinConfig
: KylinConfig.createKylinConfig(kylinConfig);
- val metadataTool = new MetadataTool(backupConfig);
- metadataTool.execute(args);
- }
-
- public void rotateAuditLog() {
- val kylinConfig = KylinConfig.getInstanceFromEnv();
- val resourceStore = ResourceStore.getKylinMetaStore(kylinConfig);
- val auditLogStore = resourceStore.getAuditLogStore();
- auditLogStore.rotate();
- }
-
- public String backupProject(String project) throws IOException {
- val folder = LocalDateTime.now(Clock.systemDefaultZone()).format(MetadataTool.DATE_TIME_FORMATTER) + "_backup";
- String[] args = new String[] { "-backup", "-compress", "-project", project, "-folder", folder, "-dir",
- getBackupDir() };
- backup(args);
- return StringUtils.appendIfMissing(getBackupDir(), "/") + folder;
+ String backupDir = getBackupDir(kylinConfig);
+ helper.backup(backupConfig, project, backupDir, folder, true, false);
+ return StringUtils.appendIfMissing(backupDir, "/") + folder;
}
- private String getBackupDir() {
- return HadoopUtil.getBackupFolder(KylinConfig.getInstanceFromEnv());
+ private String getBackupDir(KylinConfig kylinConfig) {
+ return HadoopUtil.getBackupFolder(kylinConfig);
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index 3658763858..393d7a0653 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -948,7 +948,7 @@ public class ProjectService extends BasicService {
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#project, 'ADMINISTRATION')")
- public String backupProject(String project) throws IOException {
+ public String backupProject(String project) throws Exception {
return metadataBackupService.backupProject(project);
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
index 4eda8de07e..63ddcda042 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/SystemService.java
@@ -29,7 +29,6 @@ import static org.apache.kylin.tool.constant.StageEnum.DONE;
import java.io.File;
import java.io.IOException;
-import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
@@ -53,6 +52,7 @@ import org.apache.kylin.common.persistence.transaction.MessageSynchronization;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.BufferedLogger;
import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.NExecutableManager;
import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
@@ -66,7 +66,6 @@ import org.apache.kylin.rest.request.DiagProgressRequest;
import org.apache.kylin.rest.response.DiagStatusResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.util.AclEvaluate;
-import org.apache.kylin.tool.MetadataTool;
import org.apache.kylin.tool.constant.DiagTypeEnum;
import org.apache.kylin.tool.constant.StageEnum;
import org.slf4j.Logger;
@@ -78,7 +77,6 @@ import org.springframework.stereotype.Service;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
-import com.google.common.collect.Lists;
import lombok.Data;
import lombok.NoArgsConstructor;
@@ -89,6 +87,7 @@ public class SystemService extends BasicService {
private static final Logger logger = LoggerFactory.getLogger(SystemService.class);
+ private final MetadataToolHelper helper = new MetadataToolHelper();
@Autowired
private AclEvaluate aclEvaluate;
@@ -112,34 +111,17 @@ public class SystemService extends BasicService {
}
}
- private Cache<String, DiagInfo> diagMap = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.DAYS).build();
- private Cache<String, DiagStatusResponse> exceptionMap = CacheBuilder.newBuilder()
+ private final Cache<String, DiagInfo> diagMap = CacheBuilder.newBuilder().expireAfterAccess(1, TimeUnit.DAYS).build();
+ private final Cache<String, DiagStatusResponse> exceptionMap = CacheBuilder.newBuilder()
.expireAfterAccess(1, TimeUnit.DAYS).build();
- private ExecutorService executorService = Executors.newSingleThreadExecutor();
+ private final ExecutorService executorService = Executors.newSingleThreadExecutor();
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#backupRequest.getProject(), 'ADMINISTRATION')")
public void backup(BackupRequest backupRequest) throws Exception {
- String[] args = createBackupArgs(backupRequest);
- val metadataTool = new MetadataTool(getConfig());
- metadataTool.execute(args);
- }
-
- private String[] createBackupArgs(BackupRequest backupRequest) {
- List<String> args = Lists.newArrayList("-backup");
- if (backupRequest.isCompress()) {
- args.add("-compress");
- }
- if (StringUtils.isNotBlank(backupRequest.getBackupPath())) {
- args.add("-dir");
- args.add(backupRequest.getBackupPath());
- }
- if (StringUtils.isNotBlank(backupRequest.getProject())) {
- args.add("-project");
- args.add(backupRequest.getProject());
- }
-
- logger.info("SystemService {}", args);
- return args.toArray(new String[0]);
+ String project = StringUtils.isNotBlank(backupRequest.getProject()) ? backupRequest.getProject() : null;
+ String path = StringUtils.isNotBlank(backupRequest.getBackupPath()) ? backupRequest.getBackupPath(): null;
+ boolean compress = backupRequest.isCompress();
+ helper.backup(getConfig(), project, path, null, compress, false);
}
// @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/UserAclService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/UserAclService.java
index 379291cbdb..90641b3154 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/UserAclService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/UserAclService.java
@@ -52,7 +52,6 @@ import org.apache.kylin.rest.security.AdminUserSyncEventNotifier;
import org.apache.kylin.rest.security.ExternalAclProvider;
import org.apache.kylin.rest.security.UserAcl;
import org.apache.kylin.rest.security.UserAclManager;
-import org.apache.kylin.tool.upgrade.UpdateUserAclTool;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.security.access.AccessDeniedException;
@@ -291,10 +290,15 @@ public class UserAclService extends BasicService implements UserAclServiceSuppor
remoteRequest(eventNotifier, StringUtils.EMPTY);
}
+ private static boolean isCustomProfile() {
+ val kylinConfig = KylinConfig.getInstanceFromEnv();
+ return "custom".equals(kylinConfig.getSecurityProfile());
+ }
+
@SneakyThrows(IOException.class)
public void syncAdminUserAcl() {
val config = KylinConfig.getInstanceFromEnv();
- if (UpdateUserAclTool.isCustomProfile()) {
+ if (isCustomProfile()) {
// invoke the AdminUserAspect
userService.listAdminUsers();
} else if ("ldap".equals(config.getSecurityProfile())) {
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/HDFSMetadataTool.java b/src/common-service/src/main/java/org/apache/kylin/tool/HDFSMetadataTool.java
similarity index 98%
rename from src/tool/src/main/java/org/apache/kylin/tool/HDFSMetadataTool.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/HDFSMetadataTool.java
index 343a750aa7..9b6aceb46e 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/HDFSMetadataTool.java
+++ b/src/common-service/src/main/java/org/apache/kylin/tool/HDFSMetadataTool.java
@@ -29,6 +29,10 @@ import org.apache.kylin.common.util.HadoopUtil;
import lombok.val;
public class HDFSMetadataTool {
+
+ private HDFSMetadataTool() {
+ }
+
public static void cleanBeforeBackup(KylinConfig kylinConfig) throws IOException {
val rootMetadataBackupPath = new Path(HadoopUtil.getBackupFolder(KylinConfig.getInstanceFromEnv()));
val fs = HadoopUtil.getWorkingFileSystem();
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/constant/DiagTypeEnum.java b/src/common-service/src/main/java/org/apache/kylin/tool/constant/DiagTypeEnum.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/constant/DiagTypeEnum.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/constant/DiagTypeEnum.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/constant/StageEnum.java b/src/common-service/src/main/java/org/apache/kylin/tool/constant/StageEnum.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/constant/StageEnum.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/constant/StageEnum.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/CheckResult.java b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/CheckResult.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/daemon/CheckResult.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/daemon/CheckResult.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/CheckStateEnum.java b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/CheckStateEnum.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/daemon/CheckStateEnum.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/daemon/CheckStateEnum.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/HealthChecker.java b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/HealthChecker.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/daemon/HealthChecker.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/daemon/HealthChecker.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/KapGuardianHATask.java b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/KapGuardianHATask.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/daemon/KapGuardianHATask.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/daemon/KapGuardianHATask.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/ServiceOpLevelEnum.java b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/ServiceOpLevelEnum.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/daemon/ServiceOpLevelEnum.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/daemon/ServiceOpLevelEnum.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/Worker.java b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/Worker.java
similarity index 97%
rename from src/tool/src/main/java/org/apache/kylin/tool/daemon/Worker.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/daemon/Worker.java
index 45001a74c7..b3eb1ad3b4 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/daemon/Worker.java
+++ b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/Worker.java
@@ -40,7 +40,7 @@ public class Worker {
private static SecretKey kgSecretKey;
@Getter
- private static String KE_PID;
+ private static String kePid;
static {
int serverPort = Integer.parseInt(getKylinConfig().getServerPort());
@@ -58,7 +58,7 @@ public class Worker {
}
public synchronized void setKEPid(String pid) {
- KE_PID = pid;
+ kePid = pid;
}
public synchronized void setKgSecretKey(SecretKey secretKey) {
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/checker/AbstractHealthChecker.java b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/checker/AbstractHealthChecker.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/daemon/checker/AbstractHealthChecker.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/daemon/checker/AbstractHealthChecker.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/checker/FullGCDurationChecker.java b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/checker/FullGCDurationChecker.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/daemon/checker/FullGCDurationChecker.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/daemon/checker/FullGCDurationChecker.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/checker/KEProcessChecker.java b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/checker/KEProcessChecker.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/daemon/checker/KEProcessChecker.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/daemon/checker/KEProcessChecker.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/checker/KEStatusChecker.java b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/checker/KEStatusChecker.java
similarity index 96%
rename from src/tool/src/main/java/org/apache/kylin/tool/daemon/checker/KEStatusChecker.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/daemon/checker/KEStatusChecker.java
index 43f462a643..95896c420a 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/daemon/checker/KEStatusChecker.java
+++ b/src/common-service/src/main/java/org/apache/kylin/tool/daemon/checker/KEStatusChecker.java
@@ -40,7 +40,7 @@ import lombok.Setter;
public class KEStatusChecker extends AbstractHealthChecker {
public static final String PERMISSION_DENIED = "Check permission failed!";
- private static final Logger logger = LoggerFactory.getLogger(AbstractHealthChecker.class);
+ private static final Logger logger = LoggerFactory.getLogger(KEStatusChecker.class);
private int failCount = 0;
public KEStatusChecker() {
@@ -57,12 +57,12 @@ public class KEStatusChecker extends AbstractHealthChecker {
setKgSecretKey(SecretKeyUtil.readKGSecretKeyFromFile());
}
- if (null == getKE_PID()) {
+ if (null == getKePid()) {
setKEPid(ToolUtil.getKylinPid());
}
- return SecretKeyUtil.generateEncryptedTokenWithPid(getKgSecretKey(), getKE_PID());
+ return SecretKeyUtil.generateEncryptedTokenWithPid(getKgSecretKey(), getKePid());
} catch (Exception e) {
- logger.error("Read KG secret key from file failed.", e);
+ logger.error("Read KG secret key from file failed.");
throw e;
}
}
@@ -84,7 +84,7 @@ public class KEStatusChecker extends AbstractHealthChecker {
setKgSecretKey(null);
}
- throw new RuntimeException("Get KE health status failed: " + response.msg);
+ throw new IllegalStateException("Get KE health status failed: " + response.msg);
}
Status status = response.getData();
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/garbage/ExecutableCleaner.java b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/ExecutableCleaner.java
similarity index 95%
rename from src/tool/src/main/java/org/apache/kylin/tool/garbage/ExecutableCleaner.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/garbage/ExecutableCleaner.java
index 13a706dfad..498e0fcb35 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/garbage/ExecutableCleaner.java
+++ b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/ExecutableCleaner.java
@@ -53,10 +53,7 @@ public class ExecutableCleaner extends MetadataCleaner {
return false;
}
ExecutableState state = job.getStatus();
- if (!state.isFinalState()) {
- return false;
- }
- return true;
+ return state.isFinalState();
}).collect(Collectors.toList());
for (AbstractExecutable executable : filteredExecutables) {
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/garbage/GarbageCleaner.java b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/GarbageCleaner.java
similarity index 94%
rename from src/tool/src/main/java/org/apache/kylin/tool/garbage/GarbageCleaner.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/garbage/GarbageCleaner.java
index 33daae9e56..5a05e52f8e 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/garbage/GarbageCleaner.java
+++ b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/GarbageCleaner.java
@@ -29,15 +29,11 @@ import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.scheduler.SourceUsageUpdateNotifier;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.metadata.project.NProjectManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import lombok.val;
public class GarbageCleaner {
- private static final Logger logger = LoggerFactory.getLogger(GarbageCleaner.class);
-
private GarbageCleaner() {
}
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/garbage/IndexCleaner.java b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/IndexCleaner.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/garbage/IndexCleaner.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/garbage/IndexCleaner.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/garbage/MetadataCleaner.java b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/MetadataCleaner.java
similarity index 92%
rename from src/tool/src/main/java/org/apache/kylin/tool/garbage/MetadataCleaner.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/garbage/MetadataCleaner.java
index 6cc8c3cb39..84f04556d8 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/garbage/MetadataCleaner.java
+++ b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/MetadataCleaner.java
@@ -19,9 +19,9 @@
package org.apache.kylin.tool.garbage;
public abstract class MetadataCleaner {
- protected String project;
+ protected final String project;
- public MetadataCleaner(String project) {
+ protected MetadataCleaner(String project) {
this.project = project;
}
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/garbage/SnapshotCleaner.java b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/SnapshotCleaner.java
similarity index 96%
rename from src/tool/src/main/java/org/apache/kylin/tool/garbage/SnapshotCleaner.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/garbage/SnapshotCleaner.java
index dea8c537e5..1add64dc9c 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/garbage/SnapshotCleaner.java
+++ b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/SnapshotCleaner.java
@@ -19,6 +19,7 @@
package org.apache.kylin.tool.garbage;
import java.io.IOException;
+import java.nio.file.FileSystems;
import java.util.HashSet;
import java.util.Set;
@@ -27,9 +28,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
-import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,7 +60,7 @@ public class SnapshotCleaner extends MetadataCleaner {
}
FileSystem fs = HadoopUtil.getWorkingFileSystem();
String baseDir = config.getMetadataWorkingDirectory();
- String resourcePath = baseDir + "/" + snapshotPath;
+ String resourcePath = baseDir + FileSystems.getDefault().getSeparator() + snapshotPath;
try {
return fs.exists(new Path(resourcePath));
} catch (IOException e) {
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/garbage/SourceUsageCleaner.java b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/SourceUsageCleaner.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/garbage/SourceUsageCleaner.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/garbage/SourceUsageCleaner.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java
similarity index 92%
rename from src/tool/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java
index c54726c4f9..ff8e833bcf 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java
+++ b/src/common-service/src/main/java/org/apache/kylin/tool/garbage/StorageCleaner.java
@@ -30,7 +30,6 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
@@ -82,6 +81,7 @@ import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
+import lombok.NoArgsConstructor;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
@@ -97,17 +97,16 @@ public class StorageCleaner {
public static final String ANSI_RESET = "\u001B[0m";
private final boolean cleanup;
- private boolean timeMachineEnabled;
+ private final boolean timeMachineEnabled;
private final Collection<String> projectNames;
- private long duration;
- private KylinConfig kylinConfig;
+ private final KylinConfig kylinConfig;
// for s3 https://olapio.atlassian.net/browse/AL-3154
- private static RateLimiter rateLimiter = RateLimiter.create(Integer.MAX_VALUE);
+ private static final RateLimiter rateLimiter = RateLimiter.create(Integer.MAX_VALUE);
@Getter
- private Map<String, String> trashRecord;
- private ResourceStore resourceStore;
+ private final Map<String, String> trashRecord;
+ private final ResourceStore resourceStore;
public StorageCleaner() throws Exception {
this(true);
@@ -154,10 +153,8 @@ public class StorageCleaner {
.collect(Collectors.toList());
projects.stream().map(project -> NDataflowManager.getInstance(config, project.getName()).listAllDataflows())
- .flatMap(Collection::stream)
- .map(dataflow -> KapConfig.wrap(dataflow.getConfig()))
- .map(KapConfig::getMetadataWorkingDirectory)
- .forEach(hdfsWorkingDir -> {
+ .flatMap(Collection::stream).map(dataflow -> KapConfig.wrap(dataflow.getConfig()))
+ .map(KapConfig::getMetadataWorkingDirectory).forEach(hdfsWorkingDir -> {
val fs = HadoopUtil.getWorkingFileSystem();
allFileSystems.add(new StorageItem(FileSystemDecorator.getInstance(fs), hdfsWorkingDir));
});
@@ -167,8 +164,9 @@ public class StorageCleaner {
// For build tasks it is a project-level parameter(Higher project-level priority), but for cleaning up storage garbage,
// WRITING_CLUSTER_WORKING_DIR is a system-level parameter
if (kylinConfig.isBuildFilesSeparationEnabled()) {
- allFileSystems.add(new StorageItem(FileSystemDecorator.getInstance(HadoopUtil.getWritingClusterFileSystem()),
- config.getWritingClusterWorkingDir("")));
+ allFileSystems
+ .add(new StorageItem(FileSystemDecorator.getInstance(HadoopUtil.getWritingClusterFileSystem()),
+ config.getWritingClusterWorkingDir("")));
}
log.info("all file systems are {}", allFileSystems);
for (StorageItem allFileSystem : allFileSystems) {
@@ -203,11 +201,10 @@ public class StorageCleaner {
}
}
boolean allSuccess = cleanup();
- duration = System.currentTimeMillis() - start;
- printConsole(allSuccess);
+ printConsole(allSuccess, System.currentTimeMillis() - start);
}
- public void printConsole(boolean success) {
+ public void printConsole(boolean success, long duration) {
System.out.println(ANSI_BLUE + "Kyligence Enterprise garbage report: (cleanup=" + cleanup + ")" + ANSI_RESET);
for (StorageItem item : outdatedItems) {
System.out.println(" Storage File: " + item.getPath());
@@ -227,7 +224,7 @@ public class StorageCleaner {
}
- public void collectDeletedProject() throws IOException {
+ public void collectDeletedProject() {
val config = KylinConfig.getInstanceFromEnv();
val projects = NProjectManager.getInstance(config).listAllProjects().stream().map(ProjectInstance::getName)
.collect(Collectors.toSet());
@@ -369,7 +366,7 @@ public class StorageCleaner {
.forEach(layout -> {
activeIndexDataPath.add(getDataLayoutDir(layout));
layout.getMultiPartition().forEach(partition -> //
- activeBucketDataPath.add(getDataPartitionDir(layout, partition)));
+ activeBucketDataPath.add(getDataPartitionDir(layout, partition)));
}));
activeIndexDataPath
.forEach(path -> activeFastBitmapIndexDataPath.add(path + HadoopUtil.FAST_BITMAP_SUFFIX));
@@ -513,8 +510,9 @@ public class StorageCleaner {
}
private void collectFromHDFS(StorageItem item) throws Exception {
- val projectFolders = item.getFileSystemDecorator().listStatus(new Path(item.getPath()), path -> !path.getName().startsWith("_")
- && (this.projectNames.isEmpty() || this.projectNames.contains(path.getName())));
+ val projectFolders = item.getFileSystemDecorator().listStatus(new Path(item.getPath()),
+ path -> !path.getName().startsWith("_")
+ && (this.projectNames.isEmpty() || this.projectNames.contains(path.getName())));
for (FileStatus projectFolder : projectFolders) {
List<FileTreeNode> tableSnapshotParents = Lists.newArrayList();
val projectNode = new ProjectFileTreeNode(projectFolder.getPath().getName());
@@ -528,7 +526,8 @@ public class StorageCleaner {
val treeNode = new FileTreeNode(pair.getFirst(), projectNode);
try {
log.debug("collect files from {}", pair.getFirst());
- Stream.of(item.getFileSystemDecorator().listStatus(new Path(item.getPath(), treeNode.getRelativePath())))
+ Stream.of(item.getFileSystemDecorator()
+ .listStatus(new Path(item.getPath(), treeNode.getRelativePath())))
.forEach(x -> pair.getSecond().add(new FileTreeNode(x.getPath().getName(), treeNode)));
} catch (FileNotFoundException e) {
log.info("folder {} not found", new Path(item.getPath(), treeNode.getRelativePath()));
@@ -545,42 +544,43 @@ public class StorageCleaner {
val slot = pair.getSecond();
for (FileTreeNode node : pair.getFirst()) {
log.debug("collect from {} -> {}", node.getName(), node);
- Stream.of(item.getFileSystemDecorator().listStatus(new Path(item.getPath(), node.getRelativePath())))
+ Stream.of(
+ item.getFileSystemDecorator().listStatus(new Path(item.getPath(), node.getRelativePath())))
.forEach(x -> slot.add(new FileTreeNode(x.getPath().getName(), node)));
}
}
- collectMultiPartitions(item, projectNode);
+ projectNode.getBuckets().addAll(collectMultiPartitions(item, projectNode.getName(), projectNode.getLayouts()));
}
}
- private void collectMultiPartitions(StorageItem item, ProjectFileTreeNode projectNode) throws IOException {
- String project = projectNode.getName();
+ private List<FileTreeNode> collectMultiPartitions(StorageItem item, String project, List<FileTreeNode> layouts)
+ throws IOException {
NDataflowManager manager = NDataflowManager.getInstance(kylinConfig, project);
- Map<String, Boolean> cached = new HashMap<>();
+ FileSystemDecorator fileSystemDecorator = item.getFileSystemDecorator();
+ String itemPath = item.getPath();
+ List<FileTreeNode> result = Lists.newArrayList();
+ HashSet<String> cached = Sets.newHashSet();
// Buckets do not certainly exist.
// Only multi level partition model should do this.
- val buckets = projectNode.getBuckets();
- for (FileTreeNode node : projectNode.getLayouts()) {
- String dataflowId = node.getParent() // segment
- .getParent().getName(); // dataflow
- if (!cached.containsKey(dataflowId)) {
- NDataflow dataflow = manager.getDataflow(dataflowId);
- if (Objects.nonNull(dataflow) //
- && Objects.nonNull(dataflow.getModel()) //
- && dataflow.getModel().isMultiPartitionModel()) {
- cached.put(dataflowId, true);
- } else {
- cached.put(dataflowId, false);
- }
+ for (FileTreeNode node : layouts) {
+ String dataflowId = node.getParent().getParent().getName(); // dataflow
+ if (cached.contains(dataflowId)) {
+ continue;
}
-
- if (Boolean.TRUE.equals(cached.get(dataflowId))) {
- Stream.of(item.getFileSystemDecorator().listStatus(new Path(item.getPath(), node.getRelativePath())))
+ NDataflow dataflow = manager.getDataflow(dataflowId);
+ if (Objects.nonNull(dataflow) //
+ && Objects.nonNull(dataflow.getModel()) //
+ && dataflow.getModel().isMultiPartitionModel()) {
+ cached.add(dataflowId);
+ result.addAll(Stream.of(fileSystemDecorator.listStatus(new Path(itemPath, node.getRelativePath())))
.filter(FileStatus::isDirectory) // Essential check in case of bad design.
- .forEach(x -> buckets.add(new FileTreeNode(x.getPath().getName(), node)));
+ .map(x -> new FileTreeNode(x.getPath().getName(), node)).collect(Collectors.toList()));
+ } else {
+ cached.add(dataflowId);
}
}
+ return result;
}
@AllArgsConstructor
@@ -610,7 +610,6 @@ public class StorageCleaner {
Thread.sleep(1000);
} catch (InterruptedException ie) {
log.error("Failed to sleep!", ie);
- ie.printStackTrace();
Thread.currentThread().interrupt();
}
}
@@ -711,6 +710,7 @@ public class StorageCleaner {
}
@Data
+ @NoArgsConstructor
@AllArgsConstructor
@RequiredArgsConstructor
public static class FileTreeNode {
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/kerberos/DelegationTokenManager.java b/src/common-service/src/main/java/org/apache/kylin/tool/kerberos/DelegationTokenManager.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/kerberos/DelegationTokenManager.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/kerberos/DelegationTokenManager.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/kerberos/KerberosLoginUtil.java b/src/common-service/src/main/java/org/apache/kylin/tool/kerberos/KerberosLoginUtil.java
similarity index 71%
rename from src/tool/src/main/java/org/apache/kylin/tool/kerberos/KerberosLoginUtil.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/kerberos/KerberosLoginUtil.java
index af8ab6d18f..4b8ef3e211 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/kerberos/KerberosLoginUtil.java
+++ b/src/common-service/src/main/java/org/apache/kylin/tool/kerberos/KerberosLoginUtil.java
@@ -19,11 +19,12 @@ package org.apache.kylin.tool.kerberos;
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
@@ -72,7 +73,7 @@ public class KerberosLoginUtil {
private static final String LOGIN_FAILED_CAUSE_TIME_OUT = "(time out) can not connect to kdc server or there is fire wall in the network";
private static final boolean IS_IBM_JDK = System.getProperty("java.vendor").contains("IBM");
- public synchronized static void login(String userPrincipal, String userKeytabPath, String krb5ConfPath,
+ public static synchronized void login(String userPrincipal, String userKeytabPath, String krb5ConfPath,
Configuration conf) throws IOException {
// 1.check input parameters
if ((userPrincipal == null) || (userPrincipal.length() <= 0)) {
@@ -97,23 +98,29 @@ public class KerberosLoginUtil {
// 2.check file exsits
File userKeytabFile = new File(userKeytabPath);
+ String userKeytabFilename = "userKeytabFile(" + userKeytabFile.getAbsolutePath() + ")";
if (!userKeytabFile.exists()) {
- LOG.error("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit.");
- throw new IOException("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") does not exsit.");
+ String message = userKeytabFilename + " does not exist.";
+ LOG.error(message);
+ throw new IOException(message);
}
if (!userKeytabFile.isFile()) {
- LOG.error("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") is not a file.");
- throw new IOException("userKeytabFile(" + userKeytabFile.getAbsolutePath() + ") is not a file.");
+ String message = userKeytabFilename + " is not a file.";
+ LOG.error(message);
+ throw new IOException(message);
}
File krb5ConfFile = new File(krb5ConfPath);
+ String krb5ConfFilename = "krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ")";
if (!krb5ConfFile.exists()) {
- LOG.error("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") does not exsit.");
- throw new IOException("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") does not exsit.");
+ String message = krb5ConfFilename + " does not exist.";
+ LOG.error(message);
+ throw new IOException(message);
}
if (!krb5ConfFile.isFile()) {
- LOG.error("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") is not a file.");
- throw new IOException("krb5ConfFile(" + krb5ConfFile.getAbsolutePath() + ") is not a file.");
+ String message = krb5ConfFilename + " is not a file.";
+ LOG.error(message);
+ throw new IOException(message);
}
// 3.set and check krb5config
@@ -125,50 +132,10 @@ public class KerberosLoginUtil {
LOG.info("Login fi success!!!!!!!!!!!!!!");
}
- private static void setConfiguration(Configuration conf) throws IOException {
+ private static void setConfiguration(Configuration conf) {
UserGroupInformation.setConfiguration(conf);
}
- private static boolean checkNeedLogin(String principal) throws IOException {
- if (!UserGroupInformation.isSecurityEnabled()) {
- LOG.error(
- "UserGroupInformation is not SecurityEnabled, please check if core-site.xml exists in classpath.");
- throw new IOException(
- "UserGroupInformation is not SecurityEnabled, please check if core-site.xml exists in classpath.");
- }
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- if ((currentUser != null) && (currentUser.hasKerberosCredentials())) {
- if (checkCurrentUserCorrect(principal)) {
- LOG.info("current user is " + currentUser + "has logined.");
- if (!currentUser.isFromKeytab()) {
- LOG.error("current user is not from keytab.");
- throw new IOException("current user is not from keytab.");
- }
- return false;
- } else {
- LOG.error("current user is " + currentUser
- + "has logined. please check your enviroment , especially when it used IBM JDK or kerberos for OS count login!!");
- throw new IOException(
- "current user is " + currentUser + " has logined. And please check your enviroment!!");
- }
- }
-
- return true;
- }
-
- public static void setKrb5Config(String krb5ConfFile) throws IOException {
- Unsafe.setProperty(JAVA_SECURITY_KRB5_CONF_KEY, krb5ConfFile);
- String ret = System.getProperty(JAVA_SECURITY_KRB5_CONF_KEY);
- if (ret == null) {
- LOG.error(JAVA_SECURITY_KRB5_CONF_KEY + " is null.");
- throw new IOException(JAVA_SECURITY_KRB5_CONF_KEY + " is null.");
- }
- if (!ret.equals(krb5ConfFile)) {
- LOG.error(JAVA_SECURITY_KRB5_CONF_KEY + " is " + ret + " is not " + krb5ConfFile + ".");
- throw new IOException(JAVA_SECURITY_KRB5_CONF_KEY + " is " + ret + " is not " + krb5ConfFile + ".");
- }
- }
-
public static void setJaasFile(String principal, String keytabPath) throws IOException {
String jaasPath = new File(System.getProperty("java.io.tmpdir")) + File.separator
+ System.getProperty("user.name") + JAAS_POSTFIX;
@@ -183,8 +150,8 @@ public class KerberosLoginUtil {
}
private static void writeJaasFile(String jaasPath, String principal, String keytabPath) throws IOException {
- try (OutputStream os = new FileOutputStream(jaasPath);
- BufferedWriter writer = new BufferedWriter(
+ try (OutputStream os = Files.newOutputStream(Paths.get(jaasPath));
+ BufferedWriter writer = new BufferedWriter(
new OutputStreamWriter(os, Charset.defaultCharset().name()))) {
writer.write(getJaasConfContext(principal, keytabPath));
writer.flush();
@@ -196,9 +163,7 @@ public class KerberosLoginUtil {
private static void deleteJaasFile(String jaasPath) throws IOException {
File jaasFile = new File(jaasPath);
if (jaasFile.exists()) {
- if (!jaasFile.delete()) {
- throw new IOException("Failed to delete exists jaas file.");
- }
+ Files.delete(jaasFile.toPath());
}
}
@@ -311,30 +276,26 @@ public class KerberosLoginUtil {
}
public static void setZookeeperServerPrincipal(String zkServerPrincipal) throws IOException {
- Unsafe.setProperty(ZOOKEEPER_SERVER_PRINCIPAL_KEY, zkServerPrincipal);
- String ret = System.getProperty(ZOOKEEPER_SERVER_PRINCIPAL_KEY);
- if (ret == null) {
- LOG.error(ZOOKEEPER_SERVER_PRINCIPAL_KEY + " is null.");
- throw new IOException(ZOOKEEPER_SERVER_PRINCIPAL_KEY + " is null.");
- }
- if (!ret.equals(zkServerPrincipal)) {
- LOG.error(ZOOKEEPER_SERVER_PRINCIPAL_KEY + " is " + ret + " is not " + zkServerPrincipal + ".");
- throw new IOException(ZOOKEEPER_SERVER_PRINCIPAL_KEY + " is " + ret + " is not " + zkServerPrincipal + ".");
- }
+ setZookeeperServerPrincipal(ZOOKEEPER_SERVER_PRINCIPAL_KEY, zkServerPrincipal);
+ }
+ public static void setKrb5Config(String krb5ConfFile) throws IOException {
+ setZookeeperServerPrincipal(JAVA_SECURITY_KRB5_CONF_KEY, krb5ConfFile);
}
- @Deprecated
+
public static void setZookeeperServerPrincipal(String zkServerPrincipalKey, String zkServerPrincipal)
throws IOException {
Unsafe.setProperty(zkServerPrincipalKey, zkServerPrincipal);
String ret = System.getProperty(zkServerPrincipalKey);
if (ret == null) {
- LOG.error(zkServerPrincipalKey + " is null.");
- throw new IOException(zkServerPrincipalKey + " is null.");
+ String message = zkServerPrincipalKey + " is null.";
+ LOG.error(message);
+ throw new IOException(message);
}
if (!ret.equals(zkServerPrincipal)) {
- LOG.error(zkServerPrincipalKey + " is " + ret + " is not " + zkServerPrincipal + ".");
- throw new IOException(zkServerPrincipalKey + " is " + ret + " is not " + zkServerPrincipal + ".");
+ String message = zkServerPrincipalKey + " is " + ret + " is not " + zkServerPrincipal + ".";
+ LOG.error(message);
+ throw new IOException(message);
}
}
@@ -353,55 +314,6 @@ public class KerberosLoginUtil {
}
}
- private static void checkAuthenticateOverKrb() throws IOException {
- UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- if (loginUser == null) {
- LOG.error("current user is " + currentUser + ", but loginUser is null.");
- throw new IOException("current user is " + currentUser + ", but loginUser is null.");
- }
- if (!loginUser.equals(currentUser)) {
- LOG.error("current user is " + currentUser + ", but loginUser is " + loginUser + ".");
- throw new IOException("current user is " + currentUser + ", but loginUser is " + loginUser + ".");
- }
- if (!loginUser.hasKerberosCredentials()) {
- LOG.error("current user is " + currentUser + " has no Kerberos Credentials.");
- throw new IOException("current user is " + currentUser + " has no Kerberos Credentials.");
- }
- if (!UserGroupInformation.isLoginKeytabBased()) {
- LOG.error("current user is " + currentUser + " is not Login Keytab Based.");
- throw new IOException("current user is " + currentUser + " is not Login Keytab Based.");
- }
- }
-
- private static boolean checkCurrentUserCorrect(String principal) throws IOException {
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- if (ugi == null) {
- LOG.error("current user still null.");
- throw new IOException("current user still null.");
- }
-
- String defaultRealm = null;
- try {
- defaultRealm = KerberosUtil.getDefaultRealm();
- } catch (Exception e) {
- LOG.warn("getDefaultRealm failed.");
- throw new IOException(e);
- }
-
- if ((defaultRealm != null) && (defaultRealm.length() > 0)) {
- StringBuilder realm = new StringBuilder();
- StringBuilder principalWithRealm = new StringBuilder();
- realm.append("@").append(defaultRealm);
- if (!principal.endsWith(realm.toString())) {
- principalWithRealm.append(principal).append(realm);
- principal = principalWithRealm.toString();
- }
- }
-
- return principal.equals(ugi.getUserName());
- }
-
public static boolean checkKeyTabIsValid(String path) {
return KeyTab.getInstance(new File(path)).isValid();
}
@@ -429,8 +341,8 @@ public class KerberosLoginUtil {
* login.
*/
private static class JaasConfiguration extends javax.security.auth.login.Configuration {
- private static final Map<String, String> BASIC_JAAS_OPTIONS = new HashMap<String, String>();
- private static final Map<String, String> KEYTAB_KERBEROS_OPTIONS = new HashMap<String, String>();
+ private static final Map<String, String> BASIC_JAAS_OPTIONS = new HashMap<>();
+ private static final Map<String, String> KEYTAB_KERBEROS_OPTIONS = new HashMap<>();
private static final AppConfigurationEntry KEYTAB_KERBEROS_LOGIN = new AppConfigurationEntry(
KerberosUtil.getKrb5LoginModuleName(), AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
KEYTAB_KERBEROS_OPTIONS);
@@ -463,12 +375,12 @@ public class KerberosLoginUtil {
private final String principal;
private javax.security.auth.login.Configuration baseConfig;
- public JaasConfiguration(String loginContextName, String principal, String keytabFile) throws IOException {
+ public JaasConfiguration(String loginContextName, String principal, String keytabFile) {
this(loginContextName, principal, keytabFile, keytabFile == null || keytabFile.length() == 0);
}
- private JaasConfiguration(String loginContextName, String principal, String keytabFile, boolean useTicketCache)
- throws IOException {
+ private JaasConfiguration(String loginContextName, String principal, String keytabFile,
+ boolean useTicketCache) {
try {
this.baseConfig = javax.security.auth.login.Configuration.getConfiguration();
} catch (SecurityException e) {
@@ -484,7 +396,7 @@ public class KerberosLoginUtil {
+ " useTicketCache=" + useTicketCache + " keytabFile=" + keytabFile);
}
- private void initKerberosOption() throws IOException {
+ private void initKerberosOption() {
if (!useTicketCache) {
if (IS_IBM_JDK) {
KEYTAB_KERBEROS_OPTIONS.put("useKeytab", keytabFile);
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/util/LdapUtils.java b/src/common-service/src/main/java/org/apache/kylin/tool/util/LdapUtils.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/util/LdapUtils.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/util/LdapUtils.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/util/ProjectTemporaryTableCleanerHelper.java b/src/common-service/src/main/java/org/apache/kylin/tool/util/ProjectTemporaryTableCleanerHelper.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/util/ProjectTemporaryTableCleanerHelper.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/util/ProjectTemporaryTableCleanerHelper.java
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java b/src/common-service/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
similarity index 90%
rename from src/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
rename to src/common-service/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
index ab91162b3d..b4cccb2868 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
+++ b/src/common-service/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
@@ -23,6 +23,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.UnknownHostException;
+import java.nio.charset.Charset;
import java.util.Locale;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -35,10 +36,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.AddressUtil;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.ShellException;
-import org.apache.kylin.common.util.AddressUtil;
import org.apache.kylin.query.util.ExtractFactory;
import org.apache.spark.sql.SparderEnv;
@@ -56,19 +57,19 @@ public class ToolUtil {
public static void dumpKylinJStack(File outputFile) throws IOException, ShellException {
String jstackDumpCmd = String.format(Locale.ROOT, "jstack -l %s", getKylinPid());
val result = new CliCommandExecutor().execute(jstackDumpCmd, null);
- FileUtils.writeStringToFile(outputFile, result.getCmd());
+ FileUtils.writeStringToFile(outputFile, result.getCmd(), Charset.defaultCharset());
}
public static String getKylinPid() {
File pidFile = new File(getKylinHome(), "pid");
if (pidFile.exists()) {
try {
- return FileUtils.readFileToString(pidFile);
+ return FileUtils.readFileToString(pidFile, Charset.defaultCharset());
} catch (IOException e) {
- throw new RuntimeException("Error reading KYLIN PID file.", e);
+ throw new IllegalStateException("Error reading KYLIN PID file.", e);
}
}
- throw new RuntimeException("Cannot find KYLIN PID file.");
+ throw new IllegalStateException("Cannot find KYLIN PID file.");
}
public static String getKylinHome() {
@@ -80,7 +81,7 @@ public class ToolUtil {
if (StringUtils.isNotEmpty(path)) {
return path;
}
- throw new RuntimeException("Cannot find KYLIN_HOME.");
+ throw new IllegalStateException("Cannot find KYLIN_HOME.");
}
public static String getBinFolder() {
@@ -157,12 +158,11 @@ public class ToolUtil {
}
public static boolean waitForSparderRollUp() {
- boolean isRollUp = false;
val extractor = ExtractFactory.create();
String check = SparderEnv.rollUpEventLog();
if (StringUtils.isBlank(check)) {
log.info("Failed to roll up eventLog because the spader is closed.");
- return isRollUp;
+ return false;
}
String logDir = extractor.getSparderEvenLogDir();
ExecutorService es = Executors.newSingleThreadExecutor();
@@ -176,16 +176,19 @@ public class ToolUtil {
Thread.sleep(1000);
}
});
- if (task.get(10, TimeUnit.SECONDS)) {
+ if (Boolean.TRUE.equals(task.get(10, TimeUnit.SECONDS))) {
fs.delete(new Path(logDir, check), false);
- isRollUp = true;
+ return true;
}
+ } catch (InterruptedException e) {
+ log.warn("Sparder eventLog rollUp failed.", e);
+ Thread.currentThread().interrupt();
} catch (Exception e) {
log.warn("Sparder eventLog rollUp failed.", e);
} finally {
es.shutdown();
}
- return isRollUp;
+ return false;
}
public static boolean isPortAvailable(String ip, int port) {
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/LdapUserServiceTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/LdapUserServiceTest.java
index f897e4735a..1b07ad2985 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/LdapUserServiceTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/LdapUserServiceTest.java
@@ -40,6 +40,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.EncryptUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.helper.UpdateUserAclToolHelper;
import org.apache.kylin.metadata.usergroup.UserGroup;
import org.apache.kylin.rest.response.UserGroupResponseKI;
import org.apache.kylin.rest.security.AclPermission;
@@ -47,7 +48,6 @@ import org.apache.kylin.rest.security.UserAclManager;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclUtil;
import org.apache.kylin.rest.util.SpringContext;
-import org.apache.kylin.tool.upgrade.UpdateUserAclTool;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -255,6 +255,9 @@ public class LdapUserServiceTest extends NLocalFileMetadataTestCase {
public void testCompleteUserInfoWithNotExistUser() {
ManagedUser user = new ManagedUser("NotExist", "", false);
ldapUserService.completeUserInfo(user);
+ Assert.assertEquals("NotExist", user.getUsername());
+ Assert.assertEquals("", user.getPassword());
+ Assert.assertFalse(user.isDefaultPassword());
}
@Test
@@ -395,11 +398,11 @@ public class LdapUserServiceTest extends NLocalFileMetadataTestCase {
@Test
public void testGetLdapAdminUsers() {
- UpdateUserAclTool tool = Mockito.spy(new UpdateUserAclTool());
val properties = getTestConfig().exportToProperties();
val password = properties.getProperty("kylin.security.ldap.connection-password");
- Mockito.when(tool.getPassword(properties)).thenReturn(EncryptUtil.decrypt(password));
- Assert.assertNotNull(tool.getLdapAdminUsers());
+ UpdateUserAclToolHelper helper = Mockito.spy(UpdateUserAclToolHelper.getInstance());
+ Mockito.when(helper.getPassword(properties)).thenReturn(EncryptUtil.decrypt(password));
+ Assert.assertNotNull(helper.getLdapAdminUsers());
}
@Test
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/OpenUserServiceTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/OpenUserServiceTest.java
index ef2311de30..85852ff3ab 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/OpenUserServiceTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/OpenUserServiceTest.java
@@ -28,13 +28,13 @@ import java.util.Properties;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.helper.UpdateUserAclToolHelper;
import org.apache.kylin.rest.config.initialize.UserAclListener;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.security.AclPermission;
import org.apache.kylin.rest.security.AdminUserAspect;
import org.apache.kylin.rest.security.UserAclManager;
import org.apache.kylin.rest.util.SpringContext;
-import org.apache.kylin.tool.upgrade.UpdateUserAclTool;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -143,9 +143,8 @@ public class OpenUserServiceTest extends NLocalFileMetadataTestCase {
//test list admin
getTestConfig().setProperty("kylin.security.profile", "custom");
val adminUserAspect = SpringContext.getBean(AdminUserAspect.class);
- ReflectionTestUtils.setField(adminUserAspect, "tool", Mockito.spy(new UpdateUserAclTool()));
- val tool = (UpdateUserAclTool) ReflectionTestUtils.getField(adminUserAspect, "tool");
- Mockito.when(tool.isUpgraded()).thenReturn(true);
+ UpdateUserAclToolHelper helper = Mockito.spy(UpdateUserAclToolHelper.getInstance());
+ Mockito.when(helper.isUpgraded()).thenReturn(true);
adminUserAspect.doAfterListAdminUsers(Collections.emptyList());
Assert.assertFalse((Boolean) ReflectionTestUtils.getField(adminUserAspect, "superAdminInitialized"));
List<String> admins = userService.listAdminUsers();
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/util/ProjectTemporaryTableCleanerHelperTest.java b/src/common-service/src/test/java/org/apache/kylin/tool/util/ProjectTemporaryTableCleanerHelperTest.java
similarity index 100%
rename from src/tool/src/test/java/org/apache/kylin/tool/util/ProjectTemporaryTableCleanerHelperTest.java
rename to src/common-service/src/test/java/org/apache/kylin/tool/util/ProjectTemporaryTableCleanerHelperTest.java
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/util/ToolUtilTest.java b/src/common-service/src/test/java/org/apache/kylin/tool/util/ToolUtilTest.java
similarity index 100%
rename from src/tool/src/test/java/org/apache/kylin/tool/util/ToolUtilTest.java
rename to src/common-service/src/test/java/org/apache/kylin/tool/util/ToolUtilTest.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/Pair.java b/src/core-common/src/main/java/org/apache/kylin/common/util/Pair.java
index 102a81d0cf..2fd62b6450 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/Pair.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/Pair.java
@@ -56,7 +56,7 @@ public class Pair<T1, T2> implements Serializable {
* @return a new pair containing the passed arguments
*/
public static <T1, T2> Pair<T1, T2> newPair(T1 a, T2 b) {
- return new Pair<T1, T2>(a, b);
+ return new Pair<>(a, b);
}
private static boolean equals(Object x, Object y) {
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/util/HashFunction.java b/src/core-common/src/main/java/org/apache/kylin/tool/util/HashFunction.java
similarity index 100%
rename from src/tool/src/main/java/org/apache/kylin/tool/util/HashFunction.java
rename to src/core-common/src/main/java/org/apache/kylin/tool/util/HashFunction.java
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/util/HashFunctionTest.java b/src/core-common/src/test/java/org/apache/kylin/tool/util/HashFunctionTest.java
similarity index 100%
rename from src/tool/src/test/java/org/apache/kylin/tool/util/HashFunctionTest.java
rename to src/core-common/src/test/java/org/apache/kylin/tool/util/HashFunctionTest.java
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
index 66582f5bb8..b22dee724f 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/NExecutableManager.java
@@ -245,8 +245,6 @@ public class NExecutableManager {
}
}
- //for ut
- @VisibleForTesting
public void deleteJob(String jobId) {
checkJobCanBeDeleted(jobId);
executableDao.deleteJob(jobId);
diff --git a/src/core-metadata/pom.xml b/src/core-metadata/pom.xml
index 74c4765391..d3c0bf0c87 100644
--- a/src/core-metadata/pom.xml
+++ b/src/core-metadata/pom.xml
@@ -40,6 +40,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
<dependency>
<groupId>com.tdunning</groupId>
<artifactId>t-digest</artifactId>
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
index 689fbdc831..ce18241853 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
@@ -88,7 +88,10 @@ public class HdfsCapacityMetricsTest extends NLocalFileMetadataTestCase {
fs.mkdirs(projectPath);
fs.createNewFile(projectPath);
}
+ Assert.assertTrue(hdfsCapacityMetrics.getWorkingDirCapacity().isEmpty());
hdfsCapacityMetrics.writeHdfsMetrics();
+ Assert.assertEquals(28, hdfsCapacityMetrics.getWorkingDirCapacity().size());
+
}
@Test
diff --git a/src/job-service/src/main/java/org/apache/kylin/rest/service/ScheduleService.java b/src/job-service/src/main/java/org/apache/kylin/rest/service/ScheduleService.java
index 05b744c6fe..d78e05d64a 100644
--- a/src/job-service/src/main/java/org/apache/kylin/rest/service/ScheduleService.java
+++ b/src/job-service/src/main/java/org/apache/kylin/rest/service/ScheduleService.java
@@ -17,6 +17,7 @@
*/
package org.apache.kylin.rest.service;
+import java.util.Collections;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -30,8 +31,8 @@ import org.apache.kylin.common.metrics.MetricsGroup;
import org.apache.kylin.common.metrics.MetricsName;
import org.apache.kylin.common.util.NamedThreadFactory;
import org.apache.kylin.common.util.SetThreadName;
-import org.apache.kylin.tool.routine.FastRoutineTool;
-import org.apache.kylin.tool.routine.RoutineTool;
+import org.apache.kylin.helper.MetadataToolHelper;
+import org.apache.kylin.helper.RoutineToolHelper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
@@ -61,6 +62,7 @@ public class ScheduleService {
private long opsCronTimeout;
private static final ThreadLocal<Future<?>> CURRENT_FUTURE = new ThreadLocal<>();
+ private MetadataToolHelper metadataToolHelper = new MetadataToolHelper();
@Scheduled(cron = "${kylin.metadata.ops-cron:0 0 0 * * *}")
public void routineTask() {
@@ -74,15 +76,16 @@ public class ScheduleService {
try (SetThreadName ignored = new SetThreadName("RoutineOpsWorker")) {
if (epochManager.checkEpochOwner(EpochManager.GLOBAL)) {
executeTask(() -> backupService.backupAll(), "MetadataBackup", startTime);
- executeTask(RoutineTool::cleanQueryHistories, "QueryHistoriesCleanup", startTime);
- executeTask(RoutineTool::cleanStreamingStats, "StreamingStatsCleanup", startTime);
- executeTask(RoutineTool::deleteRawRecItems, "RawRecItemsDeletion", startTime);
- executeTask(RoutineTool::cleanGlobalSourceUsage, "SourceUsageCleanup", startTime);
+ executeTask(RoutineToolHelper::cleanQueryHistories, "QueryHistoriesCleanup", startTime);
+ executeTask(RoutineToolHelper::cleanStreamingStats, "StreamingStatsCleanup", startTime);
+ executeTask(RoutineToolHelper::deleteRawRecItems, "RawRecItemsDeletion", startTime);
+ executeTask(RoutineToolHelper::cleanGlobalSourceUsage, "SourceUsageCleanup", startTime);
executeTask(() -> projectService.cleanupAcl(), "AclCleanup", startTime);
}
executeTask(() -> projectService.garbageCleanup(getRemainingTime(startTime)), "ProjectGarbageCleanup",
startTime);
- executeTask(() -> newFastRoutineTool().execute(new String[] { "-c" }), "HdfsCleanup", startTime);
+ executeTask(() -> metadataToolHelper.cleanStorage(true, Collections.emptyList(), 0, 0), "HdfsCleanup",
+ startTime);
log.info("Finish to work, cost {}ms", System.currentTimeMillis() - startTime);
}
} catch (InterruptedException e) {
@@ -114,7 +117,4 @@ public class ScheduleService {
return opsCronTimeout - (System.currentTimeMillis() - startTime);
}
- public FastRoutineTool newFastRoutineTool() {
- return new FastRoutineTool();
- }
}
diff --git a/src/kylin-it/pom.xml b/src/kylin-it/pom.xml
index ebeb329508..091bdaaea6 100644
--- a/src/kylin-it/pom.xml
+++ b/src/kylin-it/pom.xml
@@ -102,6 +102,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-streaming</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-tool</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-linq4j</artifactId>
diff --git a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NMetaStoreController.java b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NMetaStoreController.java
index 84649c4bcb..8fad872876 100644
--- a/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NMetaStoreController.java
+++ b/src/metadata-server/src/main/java/org/apache/kylin/rest/controller/NMetaStoreController.java
@@ -161,8 +161,7 @@ public class NMetaStoreController extends NBasicController {
@ApiOperation(value = "cleanupStorage", tags = { "SM" })
@PostMapping(value = "/cleanup_storage")
@ResponseBody
- public EnvelopeResponse<String> cleanupStorage(@RequestBody StorageCleanupRequest request) throws Exception {
-
+ public EnvelopeResponse<String> cleanupStorage(@RequestBody StorageCleanupRequest request) {
metaStoreService.cleanupStorage(request.getProjectsToClean(), request.isCleanupStorage());
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, "", "");
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
index 32d3f93901..3680b15cc4 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/MetaStoreService.java
@@ -37,6 +37,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -72,6 +73,8 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.MetadataChecker;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.helper.MetadataToolHelper;
+import org.apache.kylin.helper.RoutineToolHelper;
import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
@@ -106,7 +109,6 @@ import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.source.ISourceMetadataExplorer;
import org.apache.kylin.source.SourceFactory;
-import org.apache.kylin.tool.routine.RoutineTool;
import org.apache.kylin.tool.util.HashFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -152,6 +154,8 @@ public class MetaStoreService extends BasicService {
@Autowired(required = false)
private List<ModelChangeSupporter> modelChangeSupporters = Lists.newArrayList();
+ MetadataToolHelper metadataToolHelper = new MetadataToolHelper();
+
public List<ModelPreviewResponse> getPreviewModels(String project, List<String> ids) {
aclEvaluate.checkProjectWritePermission(project);
return modelService.getManager(NDataflowManager.class, project).listAllDataflows(true).stream()
@@ -706,8 +710,7 @@ public class MetaStoreService extends BasicService {
updateIndexPlan(project, nDataModel, targetIndexPlan, hasModelOverrideProps);
addWhiteListIndex(project, modelSchemaChange, targetIndexPlan);
- importRecommendations(project, nDataModel.getUuid(), importDataModel.getUuid(),
- targetKylinConfig);
+ importRecommendations(project, nDataModel.getUuid(), importDataModel.getUuid(), targetKylinConfig);
}
} catch (Exception e) {
logger.warn("Import model {} exception", modelImport.getOriginalName(), e);
@@ -787,17 +790,14 @@ public class MetaStoreService extends BasicService {
public void cleanupMeta(String project) {
if (project.equals(UnitOfWork.GLOBAL_UNIT)) {
- RoutineTool.cleanGlobalSourceUsage();
+ RoutineToolHelper.cleanGlobalSourceUsage();
QueryHisStoreUtil.cleanQueryHistory();
} else {
- RoutineTool.cleanMetaByProject(project);
+ RoutineToolHelper.cleanMetaByProject(project);
}
}
public void cleanupStorage(String[] projectsToClean, boolean cleanupStorage) {
- RoutineTool routineTool = new RoutineTool();
- routineTool.setProjects(projectsToClean);
- routineTool.setStorageCleanup(cleanupStorage);
- routineTool.cleanStorage();
+ metadataToolHelper.cleanStorage(cleanupStorage, Arrays.asList(projectsToClean), 0D, 0);
}
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/BISyncModel.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/BISyncModel.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/BISyncModel.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/BISyncModel.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/BISyncModelConverter.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/BISyncModelConverter.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/BISyncModelConverter.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/BISyncModelConverter.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/BISyncTool.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncContext.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/SyncContext.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncContext.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/SyncContext.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/SyncModelBuilder.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/ColumnDef.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/model/ColumnDef.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/ColumnDef.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/model/ColumnDef.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/JoinTreeNode.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/model/JoinTreeNode.java
similarity index 96%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/JoinTreeNode.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/model/JoinTreeNode.java
index 0613b23d81..1547608907 100644
--- a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/JoinTreeNode.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/model/JoinTreeNode.java
@@ -17,6 +17,7 @@
*/
package org.apache.kylin.tool.bisync.model;
+import java.util.Collections;
import java.util.Deque;
import java.util.LinkedList;
import java.util.List;
@@ -50,7 +51,7 @@ public class JoinTreeNode {
*/
public List<JoinTableDesc> iteratorAsList() {
if (this.value == null) {
- return null;
+ return Collections.emptyList();
}
Deque<JoinTreeNode> nodeDeque = new LinkedList<>();
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/MeasureDef.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/model/MeasureDef.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/MeasureDef.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/model/MeasureDef.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/SyncModel.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/model/SyncModel.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/model/SyncModel.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/model/SyncModel.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDataSourceConverter.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDataSourceConverter.java
similarity index 99%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDataSourceConverter.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDataSourceConverter.java
index 6540df9238..b1860cc4f9 100644
--- a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDataSourceConverter.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDataSourceConverter.java
@@ -61,7 +61,7 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-public class TableauDataSourceConverter implements BISyncModelConverter {
+public class TableauDataSourceConverter implements BISyncModelConverter<TableauDatasourceModel> {
private static final String ODBC_CONNECTION_PROJECT_PREFIX = "PROJECT=";
private static final String ODBC_CONNECTION_MODEL_PREFIX = "CUBE=";
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceModel.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceModel.java
similarity index 95%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceModel.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceModel.java
index dbec106791..f0f22ba0fc 100644
--- a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceModel.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceModel.java
@@ -42,7 +42,7 @@ public class TableauDatasourceModel implements BISyncModel {
this.tableauDatasource = tableauDatasource;
}
- public static void dumpModelAsXML(TableauDatasource BISyncModel, OutputStream outputStream)
+ public static void dumpModelAsXML(TableauDatasource biSyncModel, OutputStream outputStream)
throws XMLStreamException, IOException {
XmlMapper xmlMapper = new XmlMapper();
XMLStreamWriter writer = xmlMapper.getFactory().getXMLOutputFactory().createXMLStreamWriter(outputStream);
@@ -50,7 +50,7 @@ public class TableauDatasourceModel implements BISyncModel {
xmlMapper.enable(SerializationFeature.INDENT_OUTPUT);
xmlMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
xmlMapper.getFactory().getXMLOutputFactory().setProperty("javax.xml.stream.isRepairingNamespaces", false);
- xmlMapper.writeValue(writer, BISyncModel);
+ xmlMapper.writeValue(writer, biSyncModel);
}
@Override
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/Aliases.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/Aliases.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/Aliases.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/Aliases.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/DrillPath.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/DrillPath.java
similarity index 80%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/DrillPath.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/DrillPath.java
index c4838da6bd..b18deb6c10 100644
--- a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/DrillPath.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/DrillPath.java
@@ -66,17 +66,4 @@ public class DrillPath {
return Objects.hash(getName(), getFields());
}
- private boolean fieldsEquals(List<String> thatFields) {
- if (getFields() == thatFields) {
- return true;
- }
- if (getFields() != null && thatFields != null && getFields().size() == thatFields.size()) {
- boolean flag = true;
- for (int i = 0; i < getFields().size() && flag; i++) {
- flag = Objects.equals(getFields().get(i), thatFields.get(i));
- }
- return flag;
- }
- return false;
- }
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/DrillPaths.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/DrillPaths.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/DrillPaths.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/DrillPaths.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/Layout.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/Layout.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/Layout.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/Layout.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/SemanticValue.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/SemanticValue.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/SemanticValue.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/SemanticValue.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/SemanticValueList.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/SemanticValueList.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/SemanticValueList.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/SemanticValueList.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/TableauConnection.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/TableauConnection.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/TableauConnection.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/TableauConnection.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/TableauDatasource.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/TableauDatasource.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/TableauDatasource.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/TableauDatasource.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/column/Calculation.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/column/Calculation.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/column/Calculation.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/column/Calculation.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/column/Column.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/column/Column.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/column/Column.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/column/Column.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Col.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Col.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Col.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Col.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Cols.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Cols.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Cols.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Cols.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Connection.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Connection.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Connection.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/Connection.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/ConnectionCustomization.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/ConnectionCustomization.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/ConnectionCustomization.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/ConnectionCustomization.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/NamedConnection.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/NamedConnection.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/NamedConnection.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/NamedConnection.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/NamedConnectionList.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/NamedConnectionList.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/NamedConnectionList.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/NamedConnectionList.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Customization.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Customization.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Customization.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Customization.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/CustomizationList.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/CustomizationList.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/CustomizationList.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/CustomizationList.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Driver.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Driver.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Driver.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Driver.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Vendor.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Vendor.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Vendor.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/customization/Vendor.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/Attribute.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/Attribute.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/Attribute.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/Attribute.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/AttributeList.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/AttributeList.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/AttributeList.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/AttributeList.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/Collation.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/Collation.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/Collation.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/Collation.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/MetadataRecord.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/MetadataRecord.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/MetadataRecord.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/MetadataRecord.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/MetadataRecordList.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/MetadataRecordList.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/MetadataRecordList.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/metadata/MetadataRecordList.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Clause.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Clause.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Clause.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Clause.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Expression.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Expression.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Expression.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Expression.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Relation.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Relation.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Relation.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/datasource/connection/relation/Relation.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/FunctionMapping.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/FunctionMapping.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/FunctionMapping.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/FunctionMapping.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/Mappings.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/Mappings.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/Mappings.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/Mappings.java
diff --git a/src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/TypeMapping.java b/src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/TypeMapping.java
similarity index 100%
rename from src/query-common/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/TypeMapping.java
rename to src/modeling-service/src/main/java/org/apache/kylin/tool/bisync/tableau/mapping/TypeMapping.java
diff --git a/src/tool/src/main/resources/bisync/tds/tableau.connector.template.xml b/src/modeling-service/src/main/resources/bisync/tds/tableau.connector.template.xml
similarity index 100%
rename from src/tool/src/main/resources/bisync/tds/tableau.connector.template.xml
rename to src/modeling-service/src/main/resources/bisync/tds/tableau.connector.template.xml
diff --git a/src/tool/src/main/resources/bisync/tds/tableau.mappings.xml b/src/modeling-service/src/main/resources/bisync/tds/tableau.mappings.xml
similarity index 100%
rename from src/tool/src/main/resources/bisync/tds/tableau.mappings.xml
rename to src/modeling-service/src/main/resources/bisync/tds/tableau.mappings.xml
diff --git a/src/tool/src/main/resources/bisync/tds/tableau.template.xml b/src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml
similarity index 100%
rename from src/tool/src/main/resources/bisync/tds/tableau.template.xml
rename to src/modeling-service/src/main/resources/bisync/tds/tableau.template.xml
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
index f4194a0be4..f1db49147f 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelTdsServiceTest.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
@@ -449,7 +450,7 @@ public class ModelTdsServiceTest extends SourceTestCase {
val modelId = "cb596712-3a09-46f8-aea1-988b43fe9b6c";
prepareBasic(project);
SyncContext syncContext = tdsService.prepareSyncContext(project, modelId, SyncContext.BI.TABLEAU_CONNECTOR_TDS,
- SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL, "localhost", 8080);
+ SyncContext.ModelElement.AGG_INDEX_AND_TABLE_INDEX_COL, "localhost", 7070);
SyncModel syncModel = tdsService.exportModel(syncContext);
TableauDatasourceModel datasource1 = (TableauDatasourceModel) BISyncTool.getBISyncModel(syncContext, syncModel);
ByteArrayOutputStream outStream4 = new ByteArrayOutputStream();
@@ -459,7 +460,8 @@ public class ModelTdsServiceTest extends SourceTestCase {
}
private String getExpectedTds(String path) throws IOException {
- return CharStreams.toString(new InputStreamReader(getClass().getResourceAsStream(path), Charsets.UTF_8));
+ return CharStreams.toString(
+ new InputStreamReader(Objects.requireNonNull(getClass().getResourceAsStream(path)), Charsets.UTF_8));
}
private void prepareBasic(String project) {
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java b/src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java
similarity index 100%
rename from src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java
rename to src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/SyncModelBuilderTest.java
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelTestUtil.java b/src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/SyncModelTestUtil.java
similarity index 100%
rename from src/tool/src/test/java/org/apache/kylin/tool/bisync/SyncModelTestUtil.java
rename to src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/SyncModelTestUtil.java
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceTest.java
similarity index 100%
rename from src/tool/src/test/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceTest.java
rename to src/modeling-service/src/test/java/org/apache/kylin/tool/bisync/tableau/TableauDatasourceTest.java
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_all_cols.tds
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_basic_inner_all_cols.tds
diff --git a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds
index f8bcbfcd62..ac3e77ff62 100644
--- a/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds
+++ b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds
@@ -3,7 +3,7 @@
<connection class="federated">
<named-connections>
<named-connection caption="localhost" name="kyligence_odbc.06xjot407mgsfe1bnnyt60p4vjuf">
- <connection class="kyligence_odbc" dbname="" odbc-connect-string-extras="PROJECT=default;CUBE=nmodel_full_measure_test" port="8080" schema="DEFAULT" server="localhost" username="ADMIN" vendor1="default" vendor2="nmodel_full_measure_test"/>
+ <connection class="kyligence_odbc" dbname="" odbc-connect-string-extras="PROJECT=default;CUBE=nmodel_full_measure_test" port="7070" schema="DEFAULT" server="localhost" username="ADMIN" vendor1="default" vendor2="nmodel_full_measure_test"/>
</named-connection>
</named-connections>
<relation join="left" type="join">
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc.tds
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc_admin.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc_admin.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc_admin.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_cc_admin.tds
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_hierarchies.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_hierarchies.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_hierarchies.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_hierarchies.tds
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_no_hierarchies.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_no_hierarchies.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_no_hierarchies.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_no_hierarchies.tds
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission.tds
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_agg_index_col.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_agg_index_col.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_agg_index_col.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_agg_index_col.tds
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_all_col.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_all_col.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_all_col.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_all_col.tds
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_no_measure.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_no_measure.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_no_measure.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector_permission_no_measure.tds
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.table_index_connector.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.table_index_connector.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.table_index_connector.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.table_index_connector.tds
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.tds b/src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.tds
similarity index 100%
rename from src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.tds
rename to src/modeling-service/src/test/resources/bisync_tableau/nmodel_full_measure_test.tds
diff --git a/src/server/src/test/java/org/apache/kylin/rest/HAConfigurationTest.java b/src/server/src/test/java/org/apache/kylin/rest/HAConfigurationTest.java
index 3b25cd6717..d0e35ad355 100644
--- a/src/server/src/test/java/org/apache/kylin/rest/HAConfigurationTest.java
+++ b/src/server/src/test/java/org/apache/kylin/rest/HAConfigurationTest.java
@@ -23,9 +23,9 @@ import javax.sql.DataSource;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil;
+import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.junit.annotation.OverwriteProp;
-import org.apache.kylin.tool.util.MetadataUtil;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -48,11 +48,13 @@ class HAConfigurationTest {
@Mock
SessionProperties sessionProperties;
+ MetadataToolHelper metadataToolHelper = new MetadataToolHelper();
+
DataSource dataSource;
@BeforeEach
public void setup() throws Exception {
- dataSource = Mockito.spy(MetadataUtil.getDataSource(getTestConfig()));
+ dataSource = Mockito.spy(metadataToolHelper.getDataSource(getTestConfig()));
ReflectionTestUtils.setField(configuration, "dataSource", dataSource);
}
diff --git a/src/systools/pom.xml b/src/systools/pom.xml
index 9e68da2eba..e69de29bb2 100644
--- a/src/systools/pom.xml
+++ b/src/systools/pom.xml
@@ -1,113 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>kylin</artifactId>
- <groupId>org.apache.kylin</groupId>
- <version>5.0.0-alpha-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
- <name>Kylin - System Tools</name>
- <artifactId>kylin-systools</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-webmvc</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-ldap</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-acl</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-test</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.javassist</groupId>
- <artifactId>javassist</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito2</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.awaitility</groupId>
- <artifactId>awaitility</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.junit.vintage</groupId>
- <artifactId>junit-vintage-engine</artifactId>
- <scope>test</scope>
- </dependency>
-
- <!--FIX ME-->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-engine-spark</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-</project>
diff --git a/src/tool/pom.xml b/src/tool/pom.xml
index d7ff907fb3..9af9d5d9ab 100644
--- a/src/tool/pom.xml
+++ b/src/tool/pom.xml
@@ -42,6 +42,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-streaming</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-common-service</artifactId>
+ </dependency>
<dependency>
<groupId>org.springframework.security</groupId>
@@ -127,6 +131,12 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-common-service</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractorTool.java b/src/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractorTool.java
index b92d652dc1..8548fea87e 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractorTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractorTool.java
@@ -667,7 +667,7 @@ public abstract class AbstractInfoExtractorTool extends ExecutableApplication {
}
protected void exportJstack(File recordTime) {
- Future jstackTask = executorService.submit(() -> {
+ Future<?> jstackTask = executorService.submit(() -> {
recordTaskStartTime(JSTACK);
JStackTool.extractJstack(exportDir);
recordTaskExecutorTimeToFile(JSTACK, recordTime);
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java b/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
index daa8aed307..91217d4e48 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/MetadataTool.java
@@ -18,67 +18,33 @@
package org.apache.kylin.tool;
-import static org.apache.kylin.common.exception.code.ErrorCodeTool.FILE_ALREADY_EXISTS;
import static org.apache.kylin.common.exception.code.ErrorCodeTool.PARAMETER_NOT_SPECIFY;
-import java.io.File;
import java.io.IOException;
-import java.net.URI;
-import java.nio.file.Paths;
-import java.time.Clock;
-import java.time.LocalDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.Collections;
-import java.util.Locale;
-import java.util.NavigableSet;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.KylinConfigBase;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.AddressUtil;
import org.apache.kylin.common.util.ExecutableApplication;
import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.common.metrics.MetricsCategory;
-import org.apache.kylin.common.metrics.MetricsGroup;
-import org.apache.kylin.common.metrics.MetricsName;
-import org.apache.kylin.common.persistence.ImageDesc;
-import org.apache.kylin.common.persistence.metadata.AuditLogStore;
-import org.apache.kylin.common.persistence.transaction.UnitOfWork;
-import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
-import org.apache.kylin.common.util.AddressUtil;
-import org.apache.kylin.common.util.MetadataChecker;
import org.apache.kylin.common.util.OptionBuilder;
+import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.Unsafe;
+import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.tool.util.ScreenPrintUtil;
import org.apache.kylin.tool.util.ToolMainWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Sets;
-
-import io.kyligence.kap.guava20.shaded.common.io.ByteSource;
-import lombok.Getter;
import lombok.val;
-import lombok.var;
public class MetadataTool extends ExecutableApplication {
- public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss",
- Locale.getDefault(Locale.Category.FORMAT));
- private static final Logger logger = LoggerFactory.getLogger("diag");
- private static final String HDFS_METADATA_URL_FORMATTER = "kylin_metadata@hdfs,path=%s";
- private static final String GLOBAL = "global";
+ private static final Logger logger = LoggerFactory.getLogger("diag");
@SuppressWarnings("static-access")
private static final Option OPERATE_BACKUP = OptionBuilder.getInstance()
@@ -119,44 +85,39 @@ public class MetadataTool extends ExecutableApplication {
private final Options options;
private final KylinConfig kylinConfig;
+ private final MetadataToolHelper helper;
- private ResourceStore resourceStore;
-
- @Getter
- private String backupPath;
-
- @Getter
- private String fetchPath;
-
- MetadataTool() {
- kylinConfig = KylinConfig.getInstanceFromEnv();
- this.options = new Options();
- initOptions();
+ public MetadataTool() {
+ this(KylinConfig.getInstanceFromEnv());
}
public MetadataTool(KylinConfig kylinConfig) {
+ this(kylinConfig, new MetadataToolHelper());
+ }
+
+ public MetadataTool(KylinConfig kylinConfig, MetadataToolHelper helper) {
this.kylinConfig = kylinConfig;
- this.options = new Options();
- initOptions();
+ this.helper = helper;
+ this.options = initOptions();
}
public static void backup(KylinConfig kylinConfig) throws IOException {
HDFSMetadataTool.cleanBeforeBackup(kylinConfig);
- String[] args = new String[] { "-backup", "-compress", "-dir", HadoopUtil.getBackupFolder(kylinConfig) };
+ String[] args = new String[]{"-backup", "-compress", "-dir", HadoopUtil.getBackupFolder(kylinConfig)};
val backupTool = new MetadataTool(kylinConfig);
backupTool.execute(args);
}
public static void backup(KylinConfig kylinConfig, String dir, String folder) throws IOException {
HDFSMetadataTool.cleanBeforeBackup(kylinConfig);
- String[] args = new String[] { "-backup", "-compress", "-dir", dir, "-folder", folder };
+ String[] args = new String[]{"-backup", "-compress", "-dir", dir, "-folder", folder};
val backupTool = new MetadataTool(kylinConfig);
backupTool.execute(args);
}
public static void restore(KylinConfig kylinConfig, String folder) throws IOException {
val tool = new MetadataTool(kylinConfig);
- tool.execute(new String[] { "-restore", "-dir", folder, "--after-truncate" });
+ tool.execute(new String[]{"-restore", "-dir", folder, "--after-truncate"});
}
public static void main(String[] args) {
@@ -173,116 +134,28 @@ public class MetadataTool extends ExecutableApplication {
val resourceStore = ResourceStore.getKylinMetaStore(config);
resourceStore.getAuditLogStore().setInstance(AddressUtil.getMockPortAddress());
tool.execute(args);
- if (isBackup && StringUtils.isNotEmpty(tool.getBackupPath())) {
- System.out.printf(Locale.ROOT, "The metadata backup path is %s.%n", tool.getBackupPath());
- }
});
Unsafe.systemExit(0);
}
- public static void restore(ResourceStore currentResourceStore, ResourceStore restoreResourceStore, String project,
- boolean delete) {
- if (StringUtils.isBlank(project)) {
- logger.info("start to restore all projects");
- var srcProjectFolders = restoreResourceStore.listResources("/");
- var destProjectFolders = currentResourceStore.listResources("/");
- srcProjectFolders = srcProjectFolders == null ? Sets.newTreeSet() : srcProjectFolders;
- destProjectFolders = destProjectFolders == null ? Sets.newTreeSet() : destProjectFolders;
- val projectFolders = Sets.union(srcProjectFolders, destProjectFolders);
-
- for (String projectPath : projectFolders) {
- if (projectPath.equals(ResourceStore.METASTORE_UUID_TAG)
- || projectPath.equals(ResourceStore.METASTORE_IMAGE)) {
- continue;
- }
- val projectName = Paths.get(projectPath).getName(0).toString();
- val destResources = currentResourceStore.listResourcesRecursively(projectPath);
- val srcResources = restoreResourceStore.listResourcesRecursively(projectPath);
- UnitOfWork.doInTransactionWithRetry(() -> doRestore(currentResourceStore, restoreResourceStore,
- destResources, srcResources, delete), projectName, 1);
- }
-
- } else {
- logger.info("start to restore project {}", project);
- val destGlobalProjectResources = currentResourceStore.listResourcesRecursively(ResourceStore.PROJECT_ROOT);
-
- Set<String> globalDestResources = null;
- if (Objects.nonNull(destGlobalProjectResources)) {
- globalDestResources = destGlobalProjectResources.stream().filter(x -> Paths.get(x).getFileName()
- .toString().equals(String.format(Locale.ROOT, "%s.json", project))).collect(Collectors.toSet());
- }
-
- val globalSrcResources = restoreResourceStore
- .listResourcesRecursively(ResourceStore.PROJECT_ROOT).stream().filter(x -> Paths.get(x)
- .getFileName().toString().equals(String.format(Locale.ROOT, "%s.json", project)))
- .collect(Collectors.toSet());
-
- Set<String> finalGlobalDestResources = globalDestResources;
-
- UnitOfWork.doInTransactionWithRetry(() -> doRestore(currentResourceStore, restoreResourceStore,
- finalGlobalDestResources, globalSrcResources, delete), UnitOfWork.GLOBAL_UNIT, 1);
-
- val projectPath = "/" + project;
- val destResources = currentResourceStore.listResourcesRecursively(projectPath);
- val srcResources = restoreResourceStore.listResourcesRecursively(projectPath);
-
- UnitOfWork.doInTransactionWithRetry(
- () -> doRestore(currentResourceStore, restoreResourceStore, destResources, srcResources, delete),
- project, 1);
- }
-
- logger.info("restore successfully");
- }
-
- private static int doRestore(ResourceStore currentResourceStore, ResourceStore restoreResourceStore,
- Set<String> destResources, Set<String> srcResources, boolean delete) throws IOException {
- val threadViewRS = ResourceStore.getKylinMetaStore(KylinConfig.getInstanceFromEnv());
-
- //check destResources and srcResources are null,because Sets.difference(srcResources, destResources) will report NullPointerException
- destResources = destResources == null ? Collections.emptySet() : destResources;
- srcResources = srcResources == null ? Collections.emptySet() : srcResources;
-
- logger.info("Start insert metadata resource...");
- val insertRes = Sets.difference(srcResources, destResources);
- for (val res : insertRes) {
- val metadataRaw = restoreResourceStore.getResource(res);
- threadViewRS.checkAndPutResource(res, metadataRaw.getByteSource(), -1L);
- }
-
- logger.info("Start update metadata resource...");
- val updateRes = Sets.intersection(destResources, srcResources);
- for (val res : updateRes) {
- val raw = currentResourceStore.getResource(res);
- val metadataRaw = restoreResourceStore.getResource(res);
- threadViewRS.checkAndPutResource(res, metadataRaw.getByteSource(), raw.getMvcc());
- }
- if (delete) {
- logger.info("Start delete metadata resource...");
- val deleteRes = Sets.difference(destResources, srcResources);
- for (val res : deleteRes) {
- threadViewRS.deleteResource(res);
- }
- }
-
- return 0;
- }
-
- private void initOptions() {
- final OptionGroup optionGroup = new OptionGroup();
+ private Options initOptions() {
+ Options result = new Options();
+ OptionGroup optionGroup = new OptionGroup();
optionGroup.setRequired(true);
optionGroup.addOption(OPERATE_BACKUP);
optionGroup.addOption(OPERATE_FETCH);
optionGroup.addOption(OPERATE_LIST);
optionGroup.addOption(OPERATE_RESTORE);
- options.addOptionGroup(optionGroup);
- options.addOption(OPTION_DIR);
- options.addOption(OPTION_PROJECT);
- options.addOption(FOLDER_NAME);
- options.addOption(OPTION_TARGET);
- options.addOption(OPERATE_COMPRESS);
- options.addOption(OPTION_EXCLUDE_TABLE_EXD);
- options.addOption(OPTION_AFTER_TRUNCATE);
+ result.addOptionGroup(optionGroup);
+ result.addOption(OPTION_DIR);
+ result.addOption(OPTION_PROJECT);
+ result.addOption(FOLDER_NAME);
+ result.addOption(OPTION_TARGET);
+ result.addOption(OPERATE_COMPRESS);
+ result.addOption(OPTION_EXCLUDE_TABLE_EXD);
+ result.addOption(OPTION_AFTER_TRUNCATE);
+ return result;
}
@Override
@@ -293,273 +166,24 @@ public class MetadataTool extends ExecutableApplication {
@Override
protected void execute(OptionsHelper optionsHelper) throws Exception {
logger.info("start to init ResourceStore");
- resourceStore = ResourceStore.getKylinMetaStore(kylinConfig);
+ String project = optionsHelper.getOptionValue(OPTION_PROJECT);
+ String path = optionsHelper.getOptionValue(OPTION_DIR);
+ String folder = optionsHelper.getOptionValue(FOLDER_NAME);
+ String target = optionsHelper.getOptionValue(OPTION_TARGET);
+ boolean compress = optionsHelper.hasOption(OPERATE_COMPRESS);
+ boolean excludeTableExd = optionsHelper.hasOption(OPTION_EXCLUDE_TABLE_EXD);
if (optionsHelper.hasOption(OPERATE_BACKUP)) {
- boolean isGlobal = null == optionsHelper.getOptionValue(OPTION_PROJECT);
- long startAt = System.currentTimeMillis();
-
- try {
- backup(optionsHelper);
- } catch (Exception be) {
- if (isGlobal) {
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_FAILED, MetricsCategory.GLOBAL, GLOBAL);
- } else {
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_FAILED, MetricsCategory.PROJECT,
- optionsHelper.getOptionValue(OPTION_PROJECT));
- }
- throw be;
- } finally {
- if (isGlobal) {
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP, MetricsCategory.GLOBAL, GLOBAL);
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_DURATION, MetricsCategory.GLOBAL, GLOBAL,
- System.currentTimeMillis() - startAt);
- } else {
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP, MetricsCategory.PROJECT,
- optionsHelper.getOptionValue(OPTION_PROJECT));
- MetricsGroup.hostTagCounterInc(MetricsName.METADATA_BACKUP_DURATION, MetricsCategory.PROJECT,
- optionsHelper.getOptionValue(OPTION_PROJECT), System.currentTimeMillis() - startAt);
- }
- }
-
+ helper.backup(kylinConfig, project, path, folder, compress, excludeTableExd);
} else if (optionsHelper.hasOption(OPERATE_FETCH)) {
- fetch(optionsHelper);
+ helper.fetch(kylinConfig, path, folder, target, excludeTableExd);
} else if (optionsHelper.hasOption(OPERATE_LIST)) {
- list(optionsHelper);
+ helper.list(kylinConfig, target);
} else if (optionsHelper.hasOption(OPERATE_RESTORE)) {
- restore(optionsHelper, optionsHelper.hasOption(OPTION_AFTER_TRUNCATE));
+ boolean delete = optionsHelper.hasOption(OPTION_AFTER_TRUNCATE);
+ helper.restore(kylinConfig, project, path, delete);
} else {
throw new KylinException(PARAMETER_NOT_SPECIFY, "-restore");
}
}
- private void abortIfAlreadyExists(String path) throws IOException {
- URI uri = HadoopUtil.makeURI(path);
- if (!uri.isAbsolute()) {
- logger.info("no scheme specified for {}, try local file system file://", path);
- File localFile = new File(path);
- if (localFile.exists()) {
- logger.error("[UNEXPECTED_THINGS_HAPPENED] local file {} already exists ", path);
- throw new KylinException(FILE_ALREADY_EXISTS, path);
- }
- return;
- }
- val fs = HadoopUtil.getWorkingFileSystem();
- if (fs.exists(new Path(path))) {
- logger.error("[UNEXPECTED_THINGS_HAPPENED] specified file {} already exists ", path);
- throw new KylinException(FILE_ALREADY_EXISTS, path);
- }
- }
-
- private void fetch(OptionsHelper optionsHelper) throws Exception {
- var path = optionsHelper.getOptionValue(OPTION_DIR);
- var folder = optionsHelper.getOptionValue(FOLDER_NAME);
- val excludeTableExd = optionsHelper.hasOption(OPTION_EXCLUDE_TABLE_EXD);
- val target = optionsHelper.getOptionValue(OPTION_TARGET);
- if (StringUtils.isBlank(path)) {
- path = KylinConfigBase.getKylinHome() + File.separator + "meta_fetch";
- }
- if (StringUtils.isEmpty(folder)) {
- folder = LocalDateTime.now(Clock.systemDefaultZone()).format(DATE_TIME_FORMATTER) + "_fetch";
- }
- if (target == null) {
- System.out.println("target file must be set with fetch mode");
- } else {
- fetchPath = StringUtils.appendIfMissing(path, "/") + folder;
- // currently do not support compress with fetch
- val fetchMetadataUrl = getMetadataUrl(fetchPath, false);
- val fetchConfig = KylinConfig.createKylinConfig(kylinConfig);
- fetchConfig.setMetadataUrl(fetchMetadataUrl);
- abortIfAlreadyExists(fetchPath);
- logger.info("The fetch metadataUrl is {} and backup path is {}", fetchMetadataUrl, fetchPath);
-
- try (val fetchResourceStore = ResourceStore.getKylinMetaStore(fetchConfig)) {
-
- val fetchMetadataStore = fetchResourceStore.getMetadataStore();
-
- String targetPath = target.startsWith("/") ? target.substring(1) : target;
-
- logger.info("start to copy target file {} from ResourceStore.", target);
- UnitOfWork.doInTransactionWithRetry(
- UnitOfWorkParams.builder().readonly(true).unitName(target).processor(() -> {
- copyResourceStore("/" + targetPath, resourceStore, fetchResourceStore, true, excludeTableExd);
- // uuid
- val uuid = resourceStore.getResource(ResourceStore.METASTORE_UUID_TAG);
- fetchResourceStore.putResourceWithoutCheck(uuid.getResPath(), uuid.getByteSource(),
- uuid.getTimestamp(), -1);
- return null;
- }).build());
- if (Thread.currentThread().isInterrupted()) {
- throw new InterruptedException("metadata task is interrupt");
- }
- logger.info("start to fetch target file {}", target);
-
- // fetchResourceStore is read-only, currently we don't do any write operation on it.
- // fetchResourceStore.deleteResource(ResourceStore.METASTORE_TRASH_RECORD);
- fetchMetadataStore.dump(fetchResourceStore);
- logger.info("fetch successfully at {}", fetchPath);
- }
- }
- }
-
- private NavigableSet<String> list(OptionsHelper optionsHelper) throws Exception {
- val target = optionsHelper.getOptionValue(OPTION_TARGET);
- var res = resourceStore.listResources(target);
- if (res == null) {
- System.out.printf("%s is not exist%n", target);
- } else {
- System.out.println("" + res);
- }
- return res;
- }
-
- private void backup(OptionsHelper optionsHelper) throws Exception {
- val project = optionsHelper.getOptionValue(OPTION_PROJECT);
- var path = optionsHelper.getOptionValue(OPTION_DIR);
- var folder = optionsHelper.getOptionValue(FOLDER_NAME);
- var compress = optionsHelper.hasOption(OPERATE_COMPRESS);
- val excludeTableExd = optionsHelper.hasOption(OPTION_EXCLUDE_TABLE_EXD);
- if (StringUtils.isBlank(path)) {
- path = KylinConfigBase.getKylinHome() + File.separator + "meta_backups";
- }
- if (StringUtils.isEmpty(folder)) {
- folder = LocalDateTime.now(Clock.systemDefaultZone()).format(DATE_TIME_FORMATTER) + "_backup";
- }
- backupPath = StringUtils.appendIfMissing(path, "/") + folder;
- val backupMetadataUrl = getMetadataUrl(backupPath, compress);
- val backupConfig = KylinConfig.createKylinConfig(kylinConfig);
- backupConfig.setMetadataUrl(backupMetadataUrl);
- abortIfAlreadyExists(backupPath);
- logger.info("The backup metadataUrl is {} and backup path is {}", backupMetadataUrl, backupPath);
-
- try (val backupResourceStore = ResourceStore.getKylinMetaStore(backupConfig)) {
-
- val backupMetadataStore = backupResourceStore.getMetadataStore();
-
- if (StringUtils.isBlank(project)) {
- logger.info("start to copy all projects from ResourceStore.");
- val auditLogStore = resourceStore.getAuditLogStore();
- long finalOffset = getOffset(auditLogStore);
- backupResourceStore.putResourceWithoutCheck(ResourceStore.METASTORE_IMAGE,
- ByteSource.wrap(JsonUtil.writeValueAsBytes(new ImageDesc(finalOffset))),
- System.currentTimeMillis(), -1);
- var projectFolders = resourceStore.listResources("/");
- if (projectFolders == null) {
- return;
- }
- UnitOfWork.doInTransactionWithRetry(() -> {
- backupProjects(projectFolders, backupResourceStore, excludeTableExd);
- return null;
- }, UnitOfWork.GLOBAL_UNIT);
-
- val uuid = resourceStore.getResource(ResourceStore.METASTORE_UUID_TAG);
- if (uuid != null) {
- backupResourceStore.putResourceWithoutCheck(uuid.getResPath(), uuid.getByteSource(),
- uuid.getTimestamp(), -1);
- }
- logger.info("start to backup all projects");
-
- } else {
- logger.info("start to copy project {} from ResourceStore.", project);
- UnitOfWork.doInTransactionWithRetry(
- UnitOfWorkParams.builder().readonly(true).unitName(project).processor(() -> {
- copyResourceStore("/" + project, resourceStore, backupResourceStore, true, excludeTableExd);
- val uuid = resourceStore.getResource(ResourceStore.METASTORE_UUID_TAG);
- backupResourceStore.putResourceWithoutCheck(uuid.getResPath(), uuid.getByteSource(),
- uuid.getTimestamp(), -1);
- return null;
- }).build());
- if (Thread.currentThread().isInterrupted()) {
- throw new InterruptedException("metadata task is interrupt");
- }
- logger.info("start to backup project {}", project);
- }
- backupResourceStore.deleteResource(ResourceStore.METASTORE_TRASH_RECORD);
- backupMetadataStore.dump(backupResourceStore);
- logger.info("backup successfully at {}", backupPath);
- }
- }
-
- private long getOffset(AuditLogStore auditLogStore) {
- long offset = 0;
- if (kylinConfig.isUTEnv())
- offset = auditLogStore.getMaxId();
- else
- offset = auditLogStore.getLogOffset() == 0 ? resourceStore.getOffset() : auditLogStore.getLogOffset();
- return offset;
- }
-
- private void backupProjects(NavigableSet<String> projectFolders, ResourceStore backupResourceStore,
- boolean excludeTableExd) throws InterruptedException {
- for (String projectPath : projectFolders) {
- if (projectPath.equals(ResourceStore.METASTORE_UUID_TAG)
- || projectPath.equals(ResourceStore.METASTORE_IMAGE)) {
- continue;
- }
- // The "_global" directory is already included in the full backup
- copyResourceStore(projectPath, resourceStore, backupResourceStore, false, excludeTableExd);
- if (Thread.currentThread().isInterrupted()) {
- throw new InterruptedException("metadata task is interrupt");
- }
- }
- }
-
- private void copyResourceStore(String projectPath, ResourceStore srcResourceStore, ResourceStore destResourceStore,
- boolean isProjectLevel, boolean excludeTableExd) {
- if (excludeTableExd) {
- String tableExdPath = projectPath + ResourceStore.TABLE_EXD_RESOURCE_ROOT;
- var projectItems = srcResourceStore.listResources(projectPath);
- for (String item : projectItems) {
- if (item.equals(tableExdPath)) {
- continue;
- }
- srcResourceStore.copy(item, destResourceStore);
- }
- } else {
- srcResourceStore.copy(projectPath, destResourceStore);
- }
- if (isProjectLevel) {
- // The project-level backup needs to contain "/_global/project/*.json"
- val projectName = Paths.get(projectPath).getFileName().toString();
- srcResourceStore.copy(ProjectInstance.concatResourcePath(projectName), destResourceStore);
- }
- }
-
- private void restore(OptionsHelper optionsHelper, boolean delete) throws IOException {
- logger.info("Restore metadata with delete : {}", delete);
- val project = optionsHelper.getOptionValue(OPTION_PROJECT);
- val restorePath = optionsHelper.getOptionValue(OPTION_DIR);
-
- val restoreMetadataUrl = getMetadataUrl(restorePath, false);
- val restoreConfig = KylinConfig.createKylinConfig(kylinConfig);
- restoreConfig.setMetadataUrl(restoreMetadataUrl);
- logger.info("The restore metadataUrl is {} and restore path is {} ", restoreMetadataUrl, restorePath);
-
- val restoreResourceStore = ResourceStore.getKylinMetaStore(restoreConfig);
- val restoreMetadataStore = restoreResourceStore.getMetadataStore();
- MetadataChecker metadataChecker = new MetadataChecker(restoreMetadataStore);
-
- val verifyResult = metadataChecker.verify();
- if (!verifyResult.isQualified()) {
- throw new RuntimeException(verifyResult.getResultMessage() + "\n the metadata dir is not qualified");
- }
- restore(resourceStore, restoreResourceStore, project, delete);
- backup(kylinConfig);
-
- }
-
- String getMetadataUrl(String rootPath, boolean compressed) {
- if (HadoopUtil.isHdfsCompatibleSchema(rootPath, kylinConfig)) {
- val url = String.format(Locale.ROOT, HDFS_METADATA_URL_FORMATTER,
- Path.getPathWithoutSchemeAndAuthority(new Path(rootPath)).toString() + "/");
- return compressed ? url + ",zip=1" : url;
-
- } else if (rootPath.startsWith("file://")) {
- rootPath = rootPath.replace("file://", "");
- return StringUtils.appendIfMissing(rootPath, "/");
-
- } else {
- return StringUtils.appendIfMissing(rootPath, "/");
-
- }
- }
}
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/RollbackTool.java b/src/tool/src/main/java/org/apache/kylin/tool/RollbackTool.java
index 0f3d737145..40073cf398 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/RollbackTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/RollbackTool.java
@@ -42,21 +42,21 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.ExecutableApplication;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.job.execution.NExecutableManager;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.common.persistence.AuditLog;
import org.apache.kylin.common.persistence.ImageDesc;
+import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.event.Event;
import org.apache.kylin.common.persistence.event.ResourceCreateOrUpdateEvent;
import org.apache.kylin.common.persistence.event.ResourceDeleteEvent;
+import org.apache.kylin.common.util.ExecutableApplication;
+import org.apache.kylin.common.util.HadoopUtil;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.MetadataChecker;
import org.apache.kylin.common.util.OptionBuilder;
+import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.Unsafe;
+import org.apache.kylin.helper.MetadataToolHelper;
+import org.apache.kylin.job.execution.NExecutableManager;
import org.apache.kylin.metadata.cube.model.NDataLayout;
import org.apache.kylin.metadata.cube.model.NDataSegDetails;
import org.apache.kylin.metadata.cube.model.NDataSegment;
@@ -67,6 +67,7 @@ import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.metadata.user.NKylinUserManager;
+import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.tool.general.RollbackStatusEnum;
import org.joda.time.format.DateTimeFormat;
@@ -81,6 +82,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class RollbackTool extends ExecutableApplication {
+ private MetadataToolHelper helper = new MetadataToolHelper();
@SuppressWarnings("static-access")
private static final String HDFS_METADATA_URL_FORMATTER = "kylin_metadata@hdfs,path=%s";
@@ -133,7 +135,6 @@ public class RollbackTool extends ExecutableApplication {
return options;
}
- @Override
protected void execute(OptionsHelper optionsHelper) throws Exception {
log.info("start roll back");
log.info("start to init ResourceStore");
@@ -227,7 +228,7 @@ public class RollbackTool extends ExecutableApplication {
long userTargetTimeMillis = formatter.parseDateTime(userTargetTime).getMillis();
long protectionTime = System.currentTimeMillis() - kylinConfig.getStorageResourceSurvivalTimeThreshold();
if (userTargetTimeMillis < protectionTime) {
- log.error("user specified time is less than protection time");
+ log.error("user specified time is less than protection time");
return false;
}
@@ -443,10 +444,9 @@ public class RollbackTool extends ExecutableApplication {
if (!verifyResult.isQualified()) {
log.error("{} \n the metadata dir is not qualified", verifyResult.getResultMessage());
}
-
- MetadataTool.restore(currentResourceStore, restoreResourceStore, project, true);
+ helper.restore(currentResourceStore, restoreResourceStore, project, true);
} catch (Exception e) {
- log.error("restore mirror resource store failed: {} ", e);
+ log.error("restore mirror resource store failed", e);
}
return true;
}
@@ -552,9 +552,9 @@ public class RollbackTool extends ExecutableApplication {
}
private String backupCurrentMetadata(KylinConfig kylinConfig) throws Exception {
- val currentBackupFolder = LocalDateTime.now(Clock.systemDefaultZone()).format(MetadataTool.DATE_TIME_FORMATTER)
+ val currentBackupFolder = LocalDateTime.now(Clock.systemDefaultZone()).format(MetadataToolHelper.DATE_TIME_FORMATTER)
+ "_backup";
- MetadataTool.backup(kylinConfig, kylinConfig.getHdfsWorkingDirectory() + "_current_backup",
+ helper.backup(kylinConfig, kylinConfig.getHdfsWorkingDirectory() + "_current_backup",
currentBackupFolder);
return currentBackupFolder;
}
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/daemon/handler/AbstractCheckStateHandler.java b/src/tool/src/main/java/org/apache/kylin/tool/daemon/handler/AbstractCheckStateHandler.java
index f7ef926b16..4773c53c09 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/daemon/handler/AbstractCheckStateHandler.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/daemon/handler/AbstractCheckStateHandler.java
@@ -48,10 +48,10 @@ public abstract class AbstractCheckStateHandler extends Worker implements CheckS
}
Preconditions.checkNotNull(getKgSecretKey(), "kg secret key is null!");
- if (null == getKE_PID()) {
+ if (null == getKePid()) {
setKEPid(ToolUtil.getKylinPid());
}
- byte[] encryptedToken = SecretKeyUtil.generateEncryptedTokenWithPid(getKgSecretKey(), getKE_PID());
+ byte[] encryptedToken = SecretKeyUtil.generateEncryptedTokenWithPid(getKgSecretKey(), getKePid());
getRestClient().downOrUpGradeKE(opLevelEnum.getOpType(), encryptedToken);
} catch (Exception e) {
logger.error("Failed to operate service {}", opLevelEnum.getOpType(), e);
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/routine/FastRoutineTool.java b/src/tool/src/main/java/org/apache/kylin/tool/routine/FastRoutineTool.java
index 1648e9ee4b..a623ef8046 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/routine/FastRoutineTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/routine/FastRoutineTool.java
@@ -18,15 +18,11 @@
package org.apache.kylin.tool.routine;
-import java.util.Arrays;
import java.util.List;
-import java.util.stream.Collectors;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.Unsafe;
-import org.apache.kylin.metadata.project.NProjectManager;
-import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.helper.RoutineToolHelper;
import org.apache.kylin.tool.MaintainModeTool;
import org.apache.kylin.tool.util.ToolMainWrapper;
@@ -42,12 +38,7 @@ public class FastRoutineTool extends RoutineTool {
return;
}
initOptionValues(optionsHelper);
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- List<ProjectInstance> instances = NProjectManager.getInstance(kylinConfig).listAllProjects();
- List<String> projectsToCleanup = Arrays.asList(getProjects());
- if (projectsToCleanup.isEmpty()) {
- projectsToCleanup = instances.stream().map(ProjectInstance::getName).collect(Collectors.toList());
- }
+ List<String> projectsToCleanup = getProjectsToCleanup();
try {
if (isMetadataCleanup()) {
System.out.println("Start to fast cleanup metadata");
@@ -57,7 +48,7 @@ public class FastRoutineTool extends RoutineTool {
if (EpochManager.getInstance().isMaintenanceMode()) {
Runtime.getRuntime().addShutdownHook(new Thread(maintainModeTool::releaseEpochs));
}
- cleanMeta(projectsToCleanup);
+ RoutineToolHelper.cleanMeta(projectsToCleanup);
}
System.out.println("Start to fast cleanup hdfs");
cleanStorage();
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/routine/RoutineTool.java b/src/tool/src/main/java/org/apache/kylin/tool/routine/RoutineTool.java
index 0eb64ae63e..a1f0500f8e 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/routine/RoutineTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/routine/RoutineTool.java
@@ -17,7 +17,6 @@
*/
package org.apache.kylin.tool.routine;
-import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
@@ -25,27 +24,18 @@ import java.util.stream.Collectors;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.ExecutableApplication;
import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.common.util.SetThreadName;
import org.apache.kylin.common.util.Unsafe;
-import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
+import org.apache.kylin.helper.MetadataToolHelper;
+import org.apache.kylin.helper.RoutineToolHelper;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.query.util.QueryHisStoreUtil;
-import org.apache.kylin.metadata.streaming.util.StreamingJobRecordStoreUtil;
-import org.apache.kylin.metadata.streaming.util.StreamingJobStatsStoreUtil;
import org.apache.kylin.tool.MaintainModeTool;
-import org.apache.kylin.tool.garbage.GarbageCleaner;
-import org.apache.kylin.tool.garbage.SourceUsageCleaner;
-import org.apache.kylin.tool.garbage.StorageCleaner;
import org.apache.kylin.tool.util.ToolMainWrapper;
-
import org.apache.kylin.metadata.epoch.EpochManager;
-import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
+
import lombok.Getter;
-import lombok.val;
import lombok.extern.slf4j.Slf4j;
@Getter
@@ -64,6 +54,8 @@ public class RoutineTool extends ExecutableApplication {
private int retryTimes;
private double requestFSRate;
+ private MetadataToolHelper helper = new MetadataToolHelper();
+
public static void main(String[] args) {
ToolMainWrapper.wrap(args, () -> {
RoutineTool tool = new RoutineTool();
@@ -73,27 +65,15 @@ public class RoutineTool extends ExecutableApplication {
}
public static void deleteRawRecItems() {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- List<ProjectInstance> projectInstances = NProjectManager.getInstance(config).listAllProjects().stream()
- .filter(projectInstance -> !projectInstance.isExpertMode()).collect(Collectors.toList());
- if (projectInstances.isEmpty()) {
- return;
- }
- try (SetThreadName ignored = new SetThreadName("DeleteRawRecItemsInDB")) {
- val jdbcRawRecStore = new JdbcRawRecStore(KylinConfig.getInstanceFromEnv());
- jdbcRawRecStore.deleteOutdated();
- } catch (Exception e) {
- log.error("delete outdated advice fail: ", e);
- }
+ RoutineToolHelper.deleteRawRecItems();
}
public static void cleanQueryHistories() {
- QueryHisStoreUtil.cleanQueryHistory();
+ RoutineToolHelper.cleanQueryHistories();
}
public static void cleanStreamingStats() {
- StreamingJobStatsStoreUtil.cleanStreamingJobStats();
- StreamingJobRecordStoreUtil.cleanStreamingJobRecord();
+ RoutineToolHelper.cleanStreamingStats();
}
@Override
@@ -108,19 +88,25 @@ public class RoutineTool extends ExecutableApplication {
return options;
}
+ protected final List<String> getProjectsToCleanup() {
+ if (getProjects().length != 0) {
+ return Arrays.asList(getProjects());
+ } else {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ List<ProjectInstance> instances = NProjectManager.getInstance(kylinConfig).listAllProjects();
+ return instances.stream().map(ProjectInstance::getName).collect(Collectors.toList());
+ }
+ }
+
+
@Override
protected void execute(OptionsHelper optionsHelper) throws Exception {
if (printUsage(optionsHelper)) {
return;
}
initOptionValues(optionsHelper);
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- List<ProjectInstance> instances = NProjectManager.getInstance(kylinConfig).listAllProjects();
System.out.println("Start to cleanup metadata");
- List<String> projectsToCleanup = Arrays.asList(projects);
- if (projectsToCleanup.isEmpty()) {
- projectsToCleanup = instances.stream().map(ProjectInstance::getName).collect(Collectors.toList());
- }
+ List<String> projectsToCleanup = getProjectsToCleanup();
MaintainModeTool maintainModeTool = new MaintainModeTool("routine tool");
maintainModeTool.init();
maintainModeTool.markEpochs();
@@ -133,7 +119,7 @@ public class RoutineTool extends ExecutableApplication {
private void doCleanup(List<String> projectsToCleanup) {
try {
if (metadataCleanup) {
- cleanMeta(projectsToCleanup);
+ RoutineToolHelper.cleanMeta(projectsToCleanup);
}
cleanStorage();
} catch (Exception e) {
@@ -141,62 +127,8 @@ public class RoutineTool extends ExecutableApplication {
}
}
- protected void cleanMeta(List<String> projectsToCleanup) throws IOException {
- try {
- cleanGlobalSourceUsage();
- for (String projName : projectsToCleanup) {
- cleanMetaByProject(projName);
- }
- cleanQueryHistories();
- cleanStreamingStats();
- deleteRawRecItems();
- System.out.println("Metadata cleanup finished");
- } catch (Exception e) {
- log.error("Metadata cleanup failed", e);
- System.out.println(StorageCleaner.ANSI_RED
- + "Metadata cleanup failed. Detailed Message is at ${KYLIN_HOME}/logs/shell.stderr"
- + StorageCleaner.ANSI_RESET);
- }
-
- }
-
- public static void cleanGlobalSourceUsage() {
- log.info("Start to clean up global meta");
- try {
- EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- new SourceUsageCleaner().cleanup();
- return null;
- }, UnitOfWork.GLOBAL_UNIT);
- } catch (Exception e) {
- log.error("Failed to clean global meta", e);
- }
- log.info("Clean up global meta finished");
-
- }
-
- public static void cleanMetaByProject(String projectName) {
- log.info("Start to clean up {} meta", projectName);
- try {
- GarbageCleaner.cleanMetadata(projectName);
- } catch (Exception e) {
- log.error("Project[{}] cleanup Metadata failed", projectName, e);
- }
- log.info("Clean up {} meta finished", projectName);
- }
-
public void cleanStorage() {
- try {
- StorageCleaner storageCleaner = new StorageCleaner(storageCleanup, Arrays.asList(projects), requestFSRate,
- retryTimes);
- System.out.println("Start to cleanup HDFS");
- storageCleaner.execute();
- System.out.println("cleanup HDFS finished");
- } catch (Exception e) {
- log.error("cleanup HDFS failed", e);
- System.out.println(StorageCleaner.ANSI_RED
- + "cleanup HDFS failed. Detailed Message is at ${KYLIN_HOME}/logs/shell.stderr"
- + StorageCleaner.ANSI_RESET);
- }
+ helper.cleanStorage(storageCleanup, Arrays.asList(projects), requestFSRate, retryTimes);
}
protected boolean printUsage(OptionsHelper optionsHelper) {
@@ -230,11 +162,5 @@ public class RoutineTool extends ExecutableApplication {
+ " Request FileSystem rate: " + requestFSRate + " Retry Times: " + retryTimes);
}
- public void setProjects(String[] projects) {
- this.projects = projects;
- }
- public void setStorageCleanup(boolean storageCleanup) {
- this.storageCleanup = storageCleanup;
- }
}
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateUserAclTool.java b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateUserAclTool.java
index e58738fd2c..4f6700d790 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateUserAclTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/upgrade/UpdateUserAclTool.java
@@ -25,15 +25,11 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
-import java.util.Map;
import java.util.Optional;
-import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import javax.naming.directory.SearchControls;
-
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.io.FileUtils;
@@ -41,14 +37,15 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.EncryptUtil;
import org.apache.kylin.common.util.ExecutableApplication;
import org.apache.kylin.common.util.OptionBuilder;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.ShellException;
+import org.apache.kylin.helper.UpdateUserAclToolHelper;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.metadata.upgrade.GlobalAclVersion;
import org.apache.kylin.metadata.upgrade.GlobalAclVersionManager;
+import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.metadata.user.NKylinUserManager;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.security.AclManager;
@@ -59,13 +56,10 @@ import org.apache.kylin.rest.security.UserAcl;
import org.apache.kylin.rest.security.UserAclManager;
import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.tool.MaintainModeTool;
-import org.apache.kylin.tool.util.LdapUtils;
import org.springframework.security.acls.domain.ConsoleAuditLogger;
import org.springframework.security.acls.model.Permission;
import org.springframework.security.acls.model.Sid;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
-import org.springframework.security.ldap.DefaultSpringSecurityContextSource;
-import org.springframework.security.ldap.SpringSecurityLdapTemplate;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -168,7 +162,7 @@ public class UpdateUserAclTool extends ExecutableApplication {
@Override
protected void execute(OptionsHelper optionsHelper) throws Exception {
- if (isUpgraded() && !optionsHelper.hasOption(OPTION_ROLLBACK)
+ if (UpdateUserAclToolHelper.getInstance().isUpgraded() && !optionsHelper.hasOption(OPTION_ROLLBACK)
&& !optionsHelper.hasOption(OPTION_FORCE_UPGRADE)) {
log.info("The acl related metadata have been upgraded.");
return;
@@ -197,11 +191,6 @@ public class UpdateUserAclTool extends ExecutableApplication {
return userAclManager.listAclUsernames().size() > 0;
}
- public boolean isUpgraded() {
- val versionManager = GlobalAclVersionManager.getInstance(KylinConfig.getInstanceFromEnv());
- return versionManager.exists();
- }
-
private Set<String> getAdminUsers() {
val config = KylinConfig.getInstanceFromEnv();
val profile = config.getSecurityProfile().toLowerCase(Locale.ROOT);
@@ -249,41 +238,11 @@ public class UpdateUserAclTool extends ExecutableApplication {
val userManager = NKylinUserManager.getInstance(KylinConfig.getInstanceFromEnv());
return userManager.list().stream()
.filter(user -> user.getAuthorities().contains(new SimpleGrantedAuthority(Constant.ROLE_ADMIN)))
- .map(user -> user.getUsername()).collect(Collectors.toList());
+ .map(ManagedUser::getUsername).collect(Collectors.toList());
}
public Set<String> getLdapAdminUsers() {
- val ldapTemplate = createLdapTemplate();
- val ldapUserDNs = LdapUtils.getAllGroupMembers(ldapTemplate,
- KylinConfig.getInstanceFromEnv().getLDAPAdminRole());
- val searchControls = new SearchControls();
- searchControls.setSearchScope(2);
- Map<String, String> dnMapperMap = LdapUtils.getAllValidUserDnMap(ldapTemplate, searchControls);
- val users = new HashSet<String>();
- for (String u : ldapUserDNs) {
- Optional.ofNullable(dnMapperMap.get(u)).ifPresent(users::add);
- }
- return users;
- }
-
- public static boolean isCustomProfile() {
- val kylinConfig = KylinConfig.getInstanceFromEnv();
- return "custom".equals(kylinConfig.getSecurityProfile());
- }
-
- private SpringSecurityLdapTemplate createLdapTemplate() {
- val properties = KylinConfig.getInstanceFromEnv().exportToProperties();
- val contextSource = new DefaultSpringSecurityContextSource(
- properties.getProperty("kylin.security.ldap.connection-server"));
- contextSource.setUserDn(properties.getProperty("kylin.security.ldap.connection-username"));
- contextSource.setPassword(getPassword(properties));
- contextSource.afterPropertiesSet();
- return new SpringSecurityLdapTemplate(contextSource);
- }
-
- public String getPassword(Properties properties) {
- val password = properties.getProperty("kylin.security.ldap.connection-password");
- return EncryptUtil.decrypt(password);
+ return UpdateUserAclToolHelper.getInstance().getLdapAdminUsers();
}
public void updateProjectAcl(String operation) {
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/util/MetadataUtil.java b/src/tool/src/main/java/org/apache/kylin/tool/util/MetadataUtil.java
index 86f649f3c6..0863295492 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/util/MetadataUtil.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/util/MetadataUtil.java
@@ -36,18 +36,18 @@ import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.ibatis.jdbc.ScriptRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.logging.LogOutputStream;
-import org.apache.kylin.common.persistence.metadata.JdbcDataSource;
import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil;
+import org.apache.kylin.helper.MetadataToolHelper;
import com.google.common.collect.Lists;
-import lombok.val;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class MetadataUtil {
private static final Charset DEFAULT_CHARSET = Charset.defaultCharset();
+ private static MetadataToolHelper metadataToolHelper = new MetadataToolHelper();
private MetadataUtil() {
}
@@ -60,10 +60,7 @@ public class MetadataUtil {
}
public static DataSource getDataSource(KylinConfig kylinConfig) throws Exception {
- val url = kylinConfig.getMetadataUrl();
- val props = JdbcUtil.datasourceParameters(url);
-
- return JdbcDataSource.getDataSource(props);
+ return metadataToolHelper.getDataSource(kylinConfig);
}
public static void createTableIfNotExist(BasicDataSource dataSource, String tableName, String tableSql,
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
index f906029aea..7db3669a90 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/MetadataToolTest.java
@@ -64,6 +64,7 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.OptionBuilder;
import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.helper.MetadataToolHelper;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.project.NProjectManager;
@@ -76,7 +77,6 @@ import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.jdbc.core.JdbcTemplate;
@@ -118,11 +118,13 @@ public class MetadataToolTest extends NLocalFileMetadataTestCase {
}
private MetadataTool tool(String path) {
- val originTool = new MetadataTool(getTestConfig());
- val tool = Mockito.spy(originTool);
- Mockito.when(tool.getMetadataUrl(Mockito.anyString(), Mockito.anyBoolean()))
- .thenReturn("kylin_metadata@hdfs,zip=1,path=file://" + path);
- return tool;
+ KylinConfig kylinConfig = getTestConfig();
+ return new MetadataTool(kylinConfig, new MetadataToolHelper() {
+ @Override
+ public String getMetadataUrl(String rootPath, boolean compressed, KylinConfig kylinConfig) {
+ return "kylin_metadata@hdfs,zip=1,path=file://" + path;
+ }
+ });
}
@Test
@@ -235,7 +237,6 @@ public class MetadataToolTest extends NLocalFileMetadataTestCase {
val archiveFolder = junitFolder.listFiles()[0];
Assertions.assertThat(archiveFolder).exists();
Assertions.assertThat(archiveFolder.list()).isNotEmpty().contains(COMPRESSED_FILE);
- Assert.assertNotNull(tool.getBackupPath());
}
private boolean assertProjectFolder(File projectFolder, File archiveFolder) {
@@ -463,15 +464,12 @@ public class MetadataToolTest extends NLocalFileMetadataTestCase {
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("demo")).isNotNull();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("ssb")).isNotNull();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("default")).isNotNull();
- val tool = tool(emptyFolder.getAbsolutePath());
+ MetadataTool tool = tool(emptyFolder.getAbsolutePath());
tool.execute(new String[] { "-restore", "-compress", "-dir", "ignored", "--after-truncate" });
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).listAllProjects()).isEmpty();
- Mockito.when(tool.getMetadataUrl(Mockito.anyString(), Mockito.anyBoolean()))
- .thenReturn("kylin_metadata@hdfs,zip=1,path=file://" + restoreFolder.getAbsolutePath());
-
+ tool = tool(restoreFolder.getAbsolutePath());
Thread.sleep(TimeUnit.SECONDS.toMillis(1));
-
tool.execute(new String[] { "-restore", "-compress", "-dir", "ignored", "--after-truncate" });
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("demo")).isNotNull();
Assertions.assertThat(NProjectManager.getInstance(getTestConfig()).getProject("ssb")).isNotNull();
@@ -702,48 +700,49 @@ public class MetadataToolTest extends NLocalFileMetadataTestCase {
@Test
public void testGetMetadataUrl() {
- val tool = new MetadataTool();
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ MetadataToolHelper metadataToolHelper = new MetadataToolHelper();
var hdfsPath = "hdfs://host/path/to/hdfs/dir";
- var hdfsMetadataUrl = tool.getMetadataUrl(hdfsPath, true);
+ var hdfsMetadataUrl = metadataToolHelper.getMetadataUrl(hdfsPath, true, kylinConfig);
Assert.assertEquals("kylin_metadata@hdfs,path=/path/to/hdfs/dir/,zip=1", hdfsMetadataUrl);
- hdfsMetadataUrl = tool.getMetadataUrl(hdfsPath, false);
+ hdfsMetadataUrl = metadataToolHelper.getMetadataUrl(hdfsPath, false, kylinConfig);
Assert.assertEquals("kylin_metadata@hdfs,path=/path/to/hdfs/dir/", hdfsMetadataUrl);
var maprfsPath = "maprfs://host/path/to/maprfs/dir";
- var maprfsMetadataUrl = tool.getMetadataUrl(maprfsPath, true);
+ var maprfsMetadataUrl = metadataToolHelper.getMetadataUrl(maprfsPath, true, kylinConfig);
Assert.assertEquals("kylin_metadata@hdfs,path=/path/to/maprfs/dir/,zip=1", maprfsMetadataUrl);
- maprfsMetadataUrl = tool.getMetadataUrl(maprfsPath, false);
+ maprfsMetadataUrl = metadataToolHelper.getMetadataUrl(maprfsPath, false, kylinConfig);
Assert.assertEquals("kylin_metadata@hdfs,path=/path/to/maprfs/dir/", maprfsMetadataUrl);
var s3Path = "s3://host/path/to/s3/dir";
- var s3MetadataUrl = tool.getMetadataUrl(s3Path, true);
+ var s3MetadataUrl = metadataToolHelper.getMetadataUrl(s3Path, true, kylinConfig);
Assert.assertEquals("kylin_metadata@hdfs,path=/path/to/s3/dir/,zip=1", s3MetadataUrl);
- s3MetadataUrl = tool.getMetadataUrl(s3Path, false);
+ s3MetadataUrl = metadataToolHelper.getMetadataUrl(s3Path, false, kylinConfig);
Assert.assertEquals("kylin_metadata@hdfs,path=/path/to/s3/dir/", s3MetadataUrl);
var s3aPath = "s3a://host/path/to/s3a/dir";
- var s3aMetadataUrl = tool.getMetadataUrl(s3aPath, true);
+ var s3aMetadataUrl = metadataToolHelper.getMetadataUrl(s3aPath, true, kylinConfig);
Assert.assertEquals("kylin_metadata@hdfs,path=/path/to/s3a/dir/,zip=1", s3aMetadataUrl);
- s3aMetadataUrl = tool.getMetadataUrl(s3aPath, false);
+ s3aMetadataUrl = metadataToolHelper.getMetadataUrl(s3aPath, false, kylinConfig);
Assert.assertEquals("kylin_metadata@hdfs,path=/path/to/s3a/dir/", s3aMetadataUrl);
var wasbPath = "wasb://host/path/to/wasb/dir";
- var wasbMetadataUrl = tool.getMetadataUrl(wasbPath, true);
+ var wasbMetadataUrl = metadataToolHelper.getMetadataUrl(wasbPath, true, kylinConfig);
Assert.assertEquals("kylin_metadata@hdfs,path=/path/to/wasb/dir/,zip=1", wasbMetadataUrl);
- wasbMetadataUrl = tool.getMetadataUrl(wasbPath, false);
+ wasbMetadataUrl = metadataToolHelper.getMetadataUrl(wasbPath, false, kylinConfig);
Assert.assertEquals("kylin_metadata@hdfs,path=/path/to/wasb/dir/", wasbMetadataUrl);
var filePath = "file:///path/to/file/dir";
- var fileMetadataUrl = tool.getMetadataUrl(filePath, true);
+ var fileMetadataUrl = metadataToolHelper.getMetadataUrl(filePath, true, kylinConfig);
Assert.assertEquals("/path/to/file/dir/", fileMetadataUrl);
- fileMetadataUrl = tool.getMetadataUrl(filePath, false);
+ fileMetadataUrl = metadataToolHelper.getMetadataUrl(filePath, false, kylinConfig);
Assert.assertEquals("/path/to/file/dir/", fileMetadataUrl);
var simplePath = "/just/a/path";
- var simpleMetadataUrl = tool.getMetadataUrl(simplePath, true);
+ var simpleMetadataUrl = metadataToolHelper.getMetadataUrl(simplePath, true, kylinConfig);
Assert.assertEquals("/just/a/path/", simpleMetadataUrl);
- simpleMetadataUrl = tool.getMetadataUrl(simplePath, false);
+ simpleMetadataUrl = metadataToolHelper.getMetadataUrl(simplePath, false, kylinConfig);
Assert.assertEquals("/just/a/path/", simpleMetadataUrl);
}
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/security/KylinPasswordResetCLITest.java b/src/tool/src/test/java/org/apache/kylin/tool/security/KylinPasswordResetCLITest.java
index e8ca0ae812..6e677dd0e4 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/security/KylinPasswordResetCLITest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/security/KylinPasswordResetCLITest.java
@@ -68,6 +68,7 @@ public class KylinPasswordResetCLITest extends LogOutputTestCase {
overwriteSystemProp("kylin.metadata.random-admin-password.enabled", "true");
val pwdEncoder = new BCryptPasswordEncoder();
overwriteSystemProp("kylin.security.user-password-encoder", pwdEncoder.getClass().getName());
+ overwriteSystemProp("kylin.metadata.random-admin-password.enabled", "true");
val user = new ManagedUser("ADMIN", "KYLIN", true, Constant.ROLE_ADMIN, Constant.GROUP_ALL_USERS);
user.setPassword(pwdEncoder.encode(user.getPassword()));
val config = KylinConfig.getInstanceFromEnv();
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/upgrade/UpdateUserAclToolTest.java b/src/tool/src/test/java/org/apache/kylin/tool/upgrade/UpdateUserAclToolTest.java
index f4b78db084..eec36ce0f3 100644
--- a/src/tool/src/test/java/org/apache/kylin/tool/upgrade/UpdateUserAclToolTest.java
+++ b/src/tool/src/test/java/org/apache/kylin/tool/upgrade/UpdateUserAclToolTest.java
@@ -20,6 +20,7 @@ package org.apache.kylin.tool.upgrade;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.helper.UpdateUserAclToolHelper;
import org.apache.kylin.rest.security.AclManager;
import org.apache.kylin.rest.security.AclPermission;
import org.apache.kylin.rest.security.UserAclManager;
@@ -56,7 +57,7 @@ public class UpdateUserAclToolTest extends NLocalFileMetadataTestCase {
Mockito.when(tool.matchUpgradeCondition(args)).thenReturn(true);
tool.execute(args);
Assert.assertTrue(tool.isAdminUserUpgraded());
- Assert.assertTrue(tool.isUpgraded());
+ Assert.assertTrue(UpdateUserAclToolHelper.getInstance().isUpgraded());
val userAclManager = UserAclManager.getInstance(getTestConfig());
Assert.assertTrue(userAclManager.get("admin_user").hasPermission(AclPermission.DATA_QUERY.getMask()));
val aclManager = createAclManager(tool);
@@ -96,7 +97,7 @@ public class UpdateUserAclToolTest extends NLocalFileMetadataTestCase {
public void testUpdateUserAcl() {
getTestConfig().setProperty("kylin.security.profile", "custom");
tool.execute(new String[] { "-f", "-s=migrate", "-v=4.5.10", "-h=." });
- Assert.assertTrue(tool.isUpgraded());
+ Assert.assertTrue(UpdateUserAclToolHelper.getInstance().isUpgraded());
}
@Test
diff --git a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds b/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds
deleted file mode 100644
index ac3e77ff62..0000000000
--- a/src/tool/src/test/resources/bisync_tableau/nmodel_full_measure_test.connector.tds
+++ /dev/null
@@ -1,125 +0,0 @@
-<?xml version='1.0' encoding='UTF-8'?>
-<datasource formatted-name="federated.0e6gjbn18cj0a41an9pi309itkyi" inline="true" source-platform="win" version="10.0">
- <connection class="federated">
- <named-connections>
- <named-connection caption="localhost" name="kyligence_odbc.06xjot407mgsfe1bnnyt60p4vjuf">
- <connection class="kyligence_odbc" dbname="" odbc-connect-string-extras="PROJECT=default;CUBE=nmodel_full_measure_test" port="7070" schema="DEFAULT" server="localhost" username="ADMIN" vendor1="default" vendor2="nmodel_full_measure_test"/>
- </named-connection>
- </named-connections>
- <relation join="left" type="join">
- <clause type="join">
- <expression op="=">
- <expression op="[TEST_MEASURE].[ID1]"/>
- <expression op="[TEST_MEASURE1].[ID1]"/>
- </expression>
- </clause>
- <relation type="table" connection="kyligence_odbc.06xjot407mgsfe1bnnyt60p4vjuf" name="TEST_MEASURE" table="[DEFAULT].[TEST_MEASURE]"/>
- <relation type="table" connection="kyligence_odbc.06xjot407mgsfe1bnnyt60p4vjuf" name="TEST_MEASURE1" table="[DEFAULT].[TEST_MEASURE1]"/>
- </relation>
- <cols>
- <map key="[FLAG (TEST_MEASURE)]" value="[TEST_MEASURE].[FLAG]"/>
- <map key="[ID2 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[ID2]"/>
- <map key="[PRICE1 (TEST_MEASURE)]" value="[TEST_MEASURE].[PRICE1]"/>
- <map key="[PRICE7 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[PRICE7]"/>
- <map key="[ID1 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[ID1]"/>
- <map key="[PRICE2 (TEST_MEASURE)]" value="[TEST_MEASURE].[PRICE2]"/>
- <map key="[FLAG (TEST_MEASURE1)]" value="[TEST_MEASURE1].[FLAG]"/>
- <map key="[ID4 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[ID4]"/>
- <map key="[PRICE3 (TEST_MEASURE)]" value="[TEST_MEASURE].[PRICE3]"/>
- <map key="[ID3 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[ID3]"/>
- <map key="[CC_PRICE8]" value="[TEST_MEASURE].[CC_PRICE8]"/>
- <map key="[CC_PRICE7]" value="[TEST_MEASURE].[CC_PRICE7]"/>
- <map key="[PRICE3 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[PRICE3]"/>
- <map key="[PRICE6 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[PRICE6]"/>
- <map key="[CC_PRICE9]" value="[TEST_MEASURE].[CC_PRICE9]"/>
- <map key="[PRICE5 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[PRICE5]"/>
- <map key="[PRICE2 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[PRICE2]"/>
- <map key="[PRICE1 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[PRICE1]"/>
- <map key="[NAME3 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[NAME3]"/>
- <map key="[PRICE5 (TEST_MEASURE)]" value="[TEST_MEASURE].[PRICE5]"/>
- <map key="[NAME4 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[NAME4]"/>
- <map key="[PRICE6 (TEST_MEASURE)]" value="[TEST_MEASURE].[PRICE6]"/>
- <map key="[NAME1 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[NAME1]"/>
- <map key="[PRICE7 (TEST_MEASURE)]" value="[TEST_MEASURE].[PRICE7]"/>
- <map key="[NAME2 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[NAME2]"/>
- <map key="[NAME2 (TEST_MEASURE)]" value="[TEST_MEASURE].[NAME2]"/>
- <map key="[ID3 (TEST_MEASURE)]" value="[TEST_MEASURE].[ID3]"/>
- <map key="[NAME3 (TEST_MEASURE)]" value="[TEST_MEASURE].[NAME3]"/>
- <map key="[ID2 (TEST_MEASURE)]" value="[TEST_MEASURE].[ID2]"/>
- <map key="[ID1 (TEST_MEASURE)]" value="[TEST_MEASURE].[ID1]"/>
- <map key="[NAME1 (TEST_MEASURE)]" value="[TEST_MEASURE].[NAME1]"/>
- <map key="[NAME4 (TEST_MEASURE)]" value="[TEST_MEASURE].[NAME4]"/>
- <map key="[ID4 (TEST_MEASURE)]" value="[TEST_MEASURE].[ID4]"/>
- <map key="[CC_PRICE3]" value="[TEST_MEASURE].[CC_PRICE3]"/>
- <map key="[CC_PRICE6]" value="[TEST_MEASURE].[CC_PRICE6]"/>
- <map key="[CC_PRICE5]" value="[TEST_MEASURE].[CC_PRICE5]"/>
- <map key="[CC_PRICE2]" value="[TEST_MEASURE].[CC_PRICE2]"/>
- <map key="[CC_PRICE1]" value="[TEST_MEASURE].[CC_PRICE1]"/>
- <map key="[TIME1 (TEST_MEASURE)]" value="[TEST_MEASURE].[TIME1]"/>
- <map key="[TIME2 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[TIME2]"/>
- <map key="[CC_PRICE10]" value="[TEST_MEASURE].[CC_PRICE10]"/>
- <map key="[TIME1 (TEST_MEASURE1)]" value="[TEST_MEASURE1].[TIME1]"/>
- <map key="[TIME2 (TEST_MEASURE)]" value="[TEST_MEASURE].[TIME2]"/>
- </cols>
- </connection>
- <aliases enabled="yes"/>
- <column caption="FLAG" datatype="boolean" name="[FLAG (TEST_MEASURE)]" role="dimension" type="nominal" hidden="true"/>
- <column caption="ID2-2" datatype="integer" name="[ID2 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="PRICE1" datatype="real" name="[PRICE1 (TEST_MEASURE)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="PRICE7" datatype="integer" name="[PRICE7 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="ID1-2" datatype="integer" name="[ID1 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="PRICE2" datatype="real" name="[PRICE2 (TEST_MEASURE)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="FLAG" datatype="boolean" name="[FLAG (TEST_MEASURE1)]" role="dimension" type="nominal" hidden="true"/>
- <column caption="ID4" datatype="integer" name="[ID4 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="PRICE3" datatype="real" name="[PRICE3 (TEST_MEASURE)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="ID3" datatype="integer" name="[ID3 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="CC_CROSSTABLE_PRICE1" datatype="integer" name="[CC_PRICE8]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="CC_PRICE7" datatype="integer" name="[CC_PRICE7]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="PRICE3" datatype="real" name="[PRICE3 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="PRICE6" datatype="integer" name="[PRICE6 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="CC_CROSSTABLE_PRICE2" datatype="integer" name="[CC_PRICE9]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="PRICE5" datatype="integer" name="[PRICE5 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="PRICE2" datatype="real" name="[PRICE2 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="PRICE1-2" datatype="real" name="[PRICE1 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="NAME3" datatype="string" name="[NAME3 (TEST_MEASURE1)]" role="dimension" type="nominal" hidden="true"/>
- <column caption="PRICE5" datatype="integer" name="[PRICE5 (TEST_MEASURE)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="NAME4" datatype="integer" name="[NAME4 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="PRICE6" datatype="integer" name="[PRICE6 (TEST_MEASURE)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="NAME1" datatype="string" name="[NAME1 (TEST_MEASURE1)]" role="dimension" type="nominal" hidden="true"/>
- <column caption="PRICE7" datatype="integer" name="[PRICE7 (TEST_MEASURE)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="NAME2" datatype="string" name="[NAME2 (TEST_MEASURE1)]" role="dimension" type="nominal" hidden="true"/>
- <column caption="NAME2" datatype="string" name="[NAME2 (TEST_MEASURE)]" role="dimension" type="nominal" hidden="true"/>
- <column caption="ID3" datatype="integer" name="[ID3 (TEST_MEASURE)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="NAME2" datatype="string" name="[NAME3 (TEST_MEASURE)]" role="dimension" type="nominal" hidden="true"/>
- <column caption="ID2" datatype="integer" name="[ID2 (TEST_MEASURE)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="ID1" datatype="integer" name="[ID1 (TEST_MEASURE)]" role="dimension" type="ordinal"/>
- <column caption="NAME1" datatype="string" name="[NAME1 (TEST_MEASURE)]" role="dimension" type="nominal" hidden="true"/>
- <column caption="NAME4" datatype="integer" name="[NAME4 (TEST_MEASURE)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="ID4" datatype="integer" name="[ID4 (TEST_MEASURE)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="CC_PRICE3" datatype="real" name="[CC_PRICE3]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="CC_PRICE6" datatype="integer" name="[CC_PRICE6]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="CC_PRICE5" datatype="integer" name="[CC_PRICE5]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="CC_PRICE2" datatype="real" name="[CC_PRICE2]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="CC_PRICE1" datatype="real" name="[CC_PRICE1]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="TIME1" datatype="date" name="[TIME1 (TEST_MEASURE)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="TIME2" datatype="datetime" name="[TIME2 (TEST_MEASURE1)]" role="dimension" type="nominal" hidden="true"/>
- <column caption="CC_PRICE10" datatype="integer" name="[CC_PRICE10]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="TIME1" datatype="date" name="[TIME1 (TEST_MEASURE1)]" role="dimension" type="ordinal" hidden="true"/>
- <column caption="TIME2" datatype="datetime" name="[TIME2 (TEST_MEASURE)]" role="dimension" type="nominal" hidden="true"/>
- <column caption="COUNT_STAR" datatype="integer" name="[COUNT_STAR]" role="measure" type="quantitative">
- <calculation class="tableau" formula="COUNT(*)"/>
- </column>
- <column caption="SUM_1" datatype="integer" name="[SUM_1]" role="measure" type="quantitative">
- <calculation class="tableau" formula="SUM(1)"/>
- </column>
- <column caption="SUM_2" datatype="real" name="[SUM_2]" role="measure" type="quantitative">
- <calculation class="tableau" formula="SUM(1.0)"/>
- </column>
- <column caption="SUM_3" datatype="real" name="[SUM_3]" role="measure" type="quantitative">
- <calculation class="tableau" formula="SUM(1.0)"/>
- </column>
- <drill-paths/>
- <semantic-values>
- <semantic-value key="[Country].[Name]" value=""美国""/>
- </semantic-values>
-</datasource>