You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2024/07/31 23:55:20 UTC
(kylin) 01/02: KYLIN-5944 Remove second storage
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3a30dcb098bb68c641eceb34ad8429d7ea4629a1
Author: lionelcao <wh...@gmail.com>
AuthorDate: Thu Apr 25 10:27:27 2024 +0800
KYLIN-5944 Remove second storage
---
build/sbin/disabled-check-3000-clickhouse.sh | 46 --
outdated/second-storage/core/pom.xml | 186 ------
.../io/kyligence/kap/secondstorage/NameUtil.java | 74 ---
.../SecondStorageConcurrentTestUtil.java | 69 ---
.../secondstorage/SecondStorageConfigLoader.java | 29 -
.../kap/secondstorage/SecondStorageConstants.java | 59 --
.../kap/secondstorage/SecondStorageLockUtils.java | 95 ---
.../kap/secondstorage/SecondStorageNodeHelper.java | 189 ------
.../kap/secondstorage/SecondStoragePlugin.java | 39 --
.../secondstorage/SecondStorageQueryRouteUtil.java | 112 ----
.../kap/secondstorage/SecondStorageUpdater.java | 61 --
.../kap/secondstorage/SecondStorageUtil.java | 665 ---------------------
.../kap/secondstorage/config/ClusterInfo.java | 174 ------
.../kap/secondstorage/config/ConfigOption.java | 85 ---
.../config/DefaultSecondStorageProperties.java | 64 --
.../kyligence/kap/secondstorage/config/Node.java | 112 ----
.../config/SecondStorageModelSegment.java | 60 --
.../config/SecondStorageProjectModelSegment.java | 41 --
.../config/SecondStorageProperties.java | 29 -
.../secondstorage/config/SecondStorageSegment.java | 38 --
.../secondstorage/database/DatabaseOperator.java | 30 -
.../kap/secondstorage/database/QueryOperator.java | 30 -
.../kap/secondstorage/ddl/AlterTable.java | 213 -------
.../kap/secondstorage/ddl/CreateDatabase.java | 47 --
.../kap/secondstorage/ddl/CreateTable.java | 76 ---
.../io/kyligence/kap/secondstorage/ddl/DDL.java | 46 --
.../kap/secondstorage/ddl/DDLOperator.java | 25 -
.../io/kyligence/kap/secondstorage/ddl/Desc.java | 39 --
.../kap/secondstorage/ddl/DropDatabase.java | 48 --
.../kyligence/kap/secondstorage/ddl/DropTable.java | 48 --
.../kap/secondstorage/ddl/ExistsDatabase.java | 37 --
.../kap/secondstorage/ddl/ExistsTable.java | 38 --
.../kap/secondstorage/ddl/InsertInto.java | 79 ---
.../kap/secondstorage/ddl/RenameTable.java | 53 --
.../io/kyligence/kap/secondstorage/ddl/Select.java | 78 ---
.../kap/secondstorage/ddl/ShowCreateDatabase.java | 37 --
.../kap/secondstorage/ddl/ShowCreateTable.java | 39 --
.../kap/secondstorage/ddl/ShowDatabases.java | 27 -
.../kap/secondstorage/ddl/ShowTables.java | 41 --
.../secondstorage/ddl/SkippingIndexChooser.java | 69 ---
.../kap/secondstorage/ddl/exp/ColumnWithAlias.java | 88 ---
.../kap/secondstorage/ddl/exp/ColumnWithType.java | 64 --
.../kap/secondstorage/ddl/exp/GroupBy.java | 35 --
.../kap/secondstorage/ddl/exp/TableIdentifier.java | 51 --
.../kap/secondstorage/ddl/visitor/BaseRender.java | 27 -
.../ddl/visitor/DefaultSQLRender.java | 326 ----------
.../secondstorage/ddl/visitor/RenderVisitor.java | 68 ---
.../kap/secondstorage/ddl/visitor/Renderable.java | 22 -
.../secondstorage/enums/LockOperateTypeEnum.java | 45 --
.../kap/secondstorage/enums/LockTypeEnum.java | 137 -----
.../kap/secondstorage/enums/SkippingIndexType.java | 55 --
.../SecondStorageDatabaseOperatorFactory.java | 25 -
.../factory/SecondStorageFactory.java | 21 -
.../factory/SecondStorageFactoryUtils.java | 70 ---
.../factory/SecondStorageIndexFactory.java | 25 -
.../factory/SecondStorageMetadataFactory.java | 25 -
.../factory/SecondStorageQueryOperatorFactory.java | 25 -
.../secondstorage/metadata/HasLayoutElement.java | 68 ---
.../kap/secondstorage/metadata/IManager.java | 31 -
.../kap/secondstorage/metadata/IManagerAware.java | 25 -
.../kap/secondstorage/metadata/Manager.java | 157 -----
.../secondstorage/metadata/MetadataOperator.java | 27 -
.../kap/secondstorage/metadata/NodeGroup.java | 133 -----
.../kap/secondstorage/metadata/PartitionType.java | 23 -
.../secondstorage/metadata/SegmentFileStatus.java | 73 ---
.../kap/secondstorage/metadata/TableData.java | 302 ----------
.../kap/secondstorage/metadata/TableEntity.java | 150 -----
.../kap/secondstorage/metadata/TableFlow.java | 212 -------
.../kap/secondstorage/metadata/TablePartition.java | 138 -----
.../kap/secondstorage/metadata/TablePlan.java | 199 ------
.../kap/secondstorage/metadata/WithLayout.java | 22 -
.../metadata/annotation/DataDefinition.java | 21 -
.../metadata/annotation/TableDefinition.java | 30 -
.../secondstorage/response/SecondStorageInfo.java | 57 --
.../secondstorage/response/SecondStorageNode.java | 70 ---
.../secondstorage/response/SizeInNodeResponse.java | 38 --
.../secondstorage/response/TableSyncResponse.java | 72 ---
.../kap/secondstorage/util/ConvertUtils.java | 117 ----
.../secondstorage/util/SecondStorageDateUtils.java | 58 --
.../secondstorage/util/SecondStorageJobUtil.java | 74 ---
.../secondstorage/util/SecondStorageSqlUtils.java | 36 --
.../apache/kylin/common/SecondStorageConfig.java | 45 --
.../main/java/org/apache/kylin/job/ProjectJob.java | 22 -
.../job/SecondStorageCleanJobBuildParams.java | 76 ---
.../kylin/job/SecondStorageCleanJobUtil.java | 35 --
.../kylin/job/SecondStorageJobParamUtil.java | 109 ----
.../org/apache/kylin/job/SecondStorageJobUtil.java | 103 ----
...econdStorageRefreshSecondaryIndexJobParams.java | 71 ---
.../SecondStorageRefreshSecondaryIndexUtil.java | 29 -
.../apache/kylin/job/SecondStorageStepFactory.java | 60 --
.../handler/AbstractSecondStorageJobHanlder.java | 36 --
.../handler/SecondStorageIndexCleanJobHandler.java | 48 --
.../handler/SecondStorageModelCleanJobHandler.java | 46 --
.../SecondStorageProjectCleanJobHandler.java | 38 --
...condStorageRefreshSecondaryIndexJobHandler.java | 50 --
.../SecondStorageSegmentCleanJobHandler.java | 47 --
.../SecondStorageSegmentLoadJobHandler.java | 115 ----
.../kap/secondstorage/ColumnMapping.scala | 23 -
.../kap/secondstorage/SecondStorage.scala | 177 ------
.../datasources/jdbc/v2/SecondStorageCatalog.scala | 74 ---
pom.xml | 127 ----
.../src/main/resources/config/config_library.csv | 7 -
.../kylin/rest/controller/NAdminController.java | 6 -
.../kylin/rest/controller/NBasicController.java | 12 -
.../rest/controller/NBasicControllerTest.java | 33 -
.../config/initialize/QueryMetricsListener.java | 3 +-
.../org/apache/kylin/rest/request/SQLRequest.java | 1 -
.../kylin/rest/response/ProjectConfigResponse.java | 5 -
.../apache/kylin/rest/service/ProjectService.java | 13 +-
.../apache/kylin/rest/service/ServiceTestBase.java | 16 +-
.../apache/kylin/common/ForceToTieredStorage.java | 26 -
.../org/apache/kylin/common/KylinConfigBase.java | 125 ----
.../java/org/apache/kylin/common/QueryContext.java | 32 -
.../kylin/common/exception/JobErrorCode.java | 4 -
.../kylin/common/exception/QueryErrorCode.java | 7 +-
.../kylin/common/exception/ServerErrorCode.java | 24 +-
.../common/exception/code/ErrorCodeServer.java | 4 -
.../org/apache/kylin/common/msg/CnMessage.java | 134 +----
.../java/org/apache/kylin/common/msg/Message.java | 115 +---
.../resources/kylin_errorcode_conf_en.properties | 3 +-
.../resources/kylin_errorcode_conf_zh.properties | 1 -
.../apache/kylin/common/KylinConfigBaseTest.java | 11 -
.../java/org/apache/kylin/job/dao/JobInfoDao.java | 2 -
.../job/execution/DefaultExecutableOnModel.java | 16 -
.../apache/kylin/job/execution/JobTypeEnum.java | 8 -
.../org/apache/kylin/job/manager/JobManager.java | 2 -
.../java/org/apache/kylin/job/model/JobParam.java | 13 +-
.../kylin/metadata/cube/model/IndexPlan.java | 18 +-
.../kylin/metadata/cube/model/NBatchConstants.java | 1 +
.../model/SegmentSecondStorageStatusEnum.java | 43 --
.../metadata/model/SegmentStatusEnumToDisplay.java | 25 +-
.../metadata/query/NativeQueryRealization.java | 2 -
.../apache/kylin/metadata/query/QueryHistory.java | 1 -
.../apache/kylin/metadata/query/QueryMetrics.java | 10 -
.../kylin/metadata/query/QueryMetricsContext.java | 14 -
.../apache/kylin/common/metrics/MetricsTag.java | 3 +-
.../src/main/resources/config/config_library.csv | 7 -
src/data-loading-service/pom.xml | 15 +-
.../apache/kylin/job/service/JobInfoService.java | 78 +--
.../kylin/rest/response/ExecutableResponse.java | 21 +-
.../org/apache/kylin/rest/service/JobService.java | 70 ---
.../kylin/job/service/JobInfoServiceTest.java | 16 +-
.../apache/kylin/rest/service/JobServiceTest.java | 107 +---
src/job-service/pom.xml | 12 +
.../rest/config/initialize/JobSchedulerTest.java | 9 +-
.../NAggIndexPriorityAnswerWithCCExprTest.java | 2 +-
.../routing/DataflowCapabilityCheckerTest.java | 285 ---------
.../query/routing/QueryLayoutChooserTest.java | 64 +-
.../query/routing/RealizationChooserTest.java | 4 +-
.../routing/TableIndexAnswerSelectStarTest.java | 11 +-
.../kylin/rest/controller/NModelController.java | 7 -
.../kylin/rest/controller/SegmentController.java | 10 +-
.../controller/open/OpenSegmentController.java | 5 +-
.../rest/controller/NModelControllerTest.java | 85 +--
.../rest/controller/SegmentControllerTest.java | 47 +-
.../controller/open/OpenSegmentControllerTest.java | 6 +-
src/modeling-service/pom.xml | 15 +-
.../kylin/rest/constant/ModelAttributeEnum.java | 10 -
.../apache/kylin/rest/request/ModelRequest.java | 3 -
.../rest/response/BuildBaseIndexResponse.java | 4 -
.../kylin/rest/response/NDataModelResponse.java | 10 -
.../kylin/rest/response/NDataSegmentResponse.java | 13 -
.../kylin/rest/service/BaseIndexUpdateHelper.java | 27 -
.../kylin/rest/service/FusionIndexService.java | 15 -
.../kylin/rest/service/IndexPlanService.java | 55 +-
.../kylin/rest/service/ModelBuildService.java | 1 -
.../kylin/rest/service/ModelSemanticHelper.java | 25 +-
.../apache/kylin/rest/service/ModelService.java | 383 +-----------
.../org/apache/kylin/rest/util/ModelUtils.java | 63 +-
.../apache/kylin/rest/service/BaseIndexTest.java | 16 -
.../kylin/rest/service/IndexPlanServiceTest.java | 2 +-
.../kylin/rest/service/ModelServiceTest.java | 466 ++++-----------
.../kylin/rest/service/ProjectServiceTest.java | 6 +-
.../kylin/rest/service/TableReloadServiceTest.java | 66 +-
.../org/apache/kylin/rest/util/ModelUtilsTest.java | 60 --
.../org.mockito.plugins.MockMaker | 1 +
.../src/main/resources/config/config_library.csv | 7 -
.../src/main/resources/config/config_library.csv | 7 -
src/query-common/pom.xml | 20 +-
.../apache/kylin/query/relnode/ContextUtil.java | 13 +-
.../org/apache/kylin/query/routing/Candidate.java | 17 -
.../query/routing/DataflowCapabilityChecker.java | 6 +-
.../kylin/query/routing/QueryLayoutChooser.java | 63 +-
.../kylin/query/routing/RealizationChooser.java | 11 -
.../org/apache/kylin/query/util/PushDownUtil.java | 4 +-
.../org/apache/kylin/query/util/QueryParams.java | 2 -
.../kylin/rest/controller/NQueryController.java | 11 -
.../rest/controller/NQueryControllerTest.java | 99 ---
.../apache/kylin/rest/service/AsyncQueryJob.java | 1 -
.../kylin/rest/service/ModelQueryService.java | 26 +-
.../kylin/rest/service/QueryHistoryScheduler.java | 44 --
.../apache/kylin/rest/service/QueryService.java | 57 --
.../rest/metrics/QueryMetricsContextTest.java | 18 -
.../kylin/rest/service/AsyncQueryJobTest.java | 5 -
.../kylin/rest/service/ModelQueryServiceTest.java | 36 +-
.../kylin/rest/service/ModelServiceQueryTest.java | 8 +-
.../rest/service/QueryMetricsListenerTest.java | 27 -
.../kylin/rest/service/QueryServiceTest.java | 187 ------
.../kylin/query/engine/exec/SparderPlanExec.java | 75 ---
.../query/engine/SparderPlanExecAbnormalTest.java | 172 ------
.../util/SecondStorageJobUtilTest.java | 99 ---
src/server/pom.xml | 23 +-
src/spark-project/engine-spark/pom.xml | 4 +
.../spark/job/NSparkCleanupAfterMergeStep.java | 23 +-
.../kylin/engine/spark/job/NSparkCubingJob.java | 98 +--
.../kylin/engine/spark/job/NSparkMergingJob.java | 47 +-
.../kylin/job/execution/step/JobStepType.java | 62 +-
src/spark-project/sparder/pom.xml | 9 -
.../kylin/query/runtime/plan/ResultPlan.scala | 11 -
.../apache/spark/sql/KylinDataFrameManager.scala | 13 +-
.../datasources/jdbc/ClickHouseDialect.scala | 217 -------
.../execution/datasources/jdbc/ShardJDBCUtil.scala | 32 -
.../jdbc/ShardJdbcRelationProvider.scala | 44 --
.../execution/datasources/jdbc/ShardOptions.scala | 52 --
.../datasources/v2/jdbc/ShardJDBCScanBuilder.scala | 46 --
.../datasources/v2/jdbc/ShardJDBCTable.scala | 32 -
.../v2/jdbc/ShardJDBCTableCatalog.scala | 89 ---
.../datasources/jdbc/ShardJDBCSuite.scala | 74 ---
.../jdbc/ShardJDBCWithoutShardSuite.scala | 106 ----
.../datasources/jdbc/v2/ShardJDBCSuite.scala | 74 ---
.../jdbc/v2/ShardJDBCWithoutShardSuite.scala | 17 -
.../kylin/tool/AbstractInfoExtractorTool.java | 11 -
.../org/apache/kylin/tool/ClickhouseDiagTool.java | 239 --------
.../java/org/apache/kylin/tool/DiagClientTool.java | 2 -
.../org/apache/kylin/tool/JobDiagInfoTool.java | 2 -
.../kylin/tool/StreamingJobDiagInfoTool.java | 2 -
.../apache/kylin/tool/ClickhouseDiagToolTest.java | 212 -------
.../org/apache/kylin/tool/DiagClientToolTest.java | 17 -
.../org/apache/kylin/tool/KylinLogToolTest.java | 2 +-
229 files changed, 350 insertions(+), 12967 deletions(-)
diff --git a/build/sbin/disabled-check-3000-clickhouse.sh b/build/sbin/disabled-check-3000-clickhouse.sh
deleted file mode 100644
index 6d911b7d85..0000000000
--- a/build/sbin/disabled-check-3000-clickhouse.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#title=Checking ClickHouse Cluster Health
-
-source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
-
-source ${KYLIN_HOME}/sbin/init-kerberos.sh
-## init Kerberos if needed
-initKerberosIfNeeded
-
-echo "Checking ClickHouse Cluster Health..."
-
-SECOND_STORAGE_CLICKHOUSE_ENABLED=`$KYLIN_HOME/bin/get-properties.sh kylin.second-storage.class`
-
-if [[ "${SECOND_STORAGE_CLICKHOUSE_ENABLED}" == "org.apache.kylin.clickhouse.ClickHouseStorage" ]]
-then
- output=`${KYLIN_HOME}/sbin/bootstrap.sh 'org.apache.kylin.clickhouse.tool.ClickHouseSanityCheckTool' 10`
- if [[ $? == 0 ]]
- then
- echo "check ClickHouse cluster access succeeded."
- exit 0
- else
- echo "check ClickHouse cluster access failed."
- exit 1
- fi
-else
- echo "Second storage is not enabled. Skip second storage cluster check."
- exit 3
-fi
diff --git a/outdated/second-storage/core/pom.xml b/outdated/second-storage/core/pom.xml
deleted file mode 100644
index 15f9f3076d..0000000000
--- a/outdated/second-storage/core/pom.xml
+++ /dev/null
@@ -1,186 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>kylin</artifactId>
- <groupId>org.apache.kylin</groupId>
- <version>5.0.0-SNAPSHOT</version>
- <relativePath>../../../pom.xml</relativePath>
- </parent>
-
- <artifactId>kap-second-storage-core</artifactId>
- <packaging>jar</packaging>
- <name>Kylin - Second Storage</name>
- <url>http://kylin.apache.org</url>
- <description>KAP Second Storage Core</description>
-
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <sonar.sources>src/main/java,src/main/scala</sonar.sources>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-job</artifactId>
- </dependency>
- <!--
- Disable lombok for scala and java mixed project.
-
- See https://stackoverflow.com/questions/11171631/error-compiling-java-scala-mixed-project-and-lombok for
- details
- -->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-spark-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </dependency>
-
- <!-- hadoop -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
-
- <!-- spark -->
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-catalyst_2.12</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-spark-common</artifactId>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.junit.vintage</groupId>
- <artifactId>junit-vintage-engine</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_2.12</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-inline</artifactId>
- <scope>test</scope>
- <version>${mockito.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-catalyst_2.12</artifactId>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-test</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>javassist</artifactId>
- <groupId>org.javassist</groupId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!--test-->
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>${powermock.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito2</artifactId>
- <version>${powermock.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-test</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-engine</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-
-</project>
\ No newline at end of file
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/NameUtil.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/NameUtil.java
deleted file mode 100644
index 304a81cebe..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/NameUtil.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage;
-
-import java.util.Locale;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.RandomUtil;
-import org.apache.kylin.metadata.cube.model.NDataflow;
-
-public class NameUtil {
-
- private static final int UUID_LENGTH = RandomUtil.randomUUIDStr().length();
- public static final String TEMP_TABLE_FLAG = "temp";
- public static final String TEMP_SRC_TABLE_FLAG = "src";
-
- private NameUtil() {
- }
-
- public static String getDatabase(NDataflow df) {
- return getDatabase(df.getConfig(), df.getProject());
- }
-
- private static String databasePrefix(KylinConfig config) {
- return config.isUTEnv() ? "UT" : config.getMetadataUrlPrefix();
- }
-
- public static String getDatabase(KylinConfig config, String project) {
- return String.format(Locale.ROOT, "%s_%s", databasePrefix(config), project);
- }
-
- public static String getTable(NDataflow df, long layoutId) {
- return getTable(df.getUuid(), layoutId);
- }
-
- public static String getTable(String modelId, long layoutId) {
- return String.format(Locale.ROOT, "%s_%d", tablePrefix(modelId), layoutId);
- }
-
- public static String tablePrefix(String modelId) {
- return modelId.replace("-", "_");
- }
-
- // reverse
- public static String recoverProject(String database, KylinConfig config) {
- return database.substring(databasePrefix(config).length() + 1);
- }
-
- public static Pair<String, Long> recoverLayout(String table) {
- String model = table.substring(0, UUID_LENGTH).replace("_", "-");
- Long layout = Long.parseLong(table.substring(UUID_LENGTH + 1));
- return Pair.newPair(model, layout);
- }
-
- public static boolean isTempTable(String table) {
- return table.contains(TEMP_TABLE_FLAG) || table.contains(TEMP_SRC_TABLE_FLAG);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageConcurrentTestUtil.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageConcurrentTestUtil.java
deleted file mode 100644
index 6bd0cc25fa..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageConcurrentTestUtil.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
-/**
- * Only used for concurrent integration test
- */
-public class SecondStorageConcurrentTestUtil {
- // for load data to clickhouse
- public static final String WAIT_PAUSED = "WAIT_PAUSED";
- public static final String WAIT_BEFORE_COMMIT = "WAIT_BEFORE_COMMIT";
- public static final String WAIT_AFTER_COMMIT = "WAIT_AFTER_COMMIT";
-
- private static final Map<String, Integer> latchMap = new ConcurrentHashMap<>();
- private static final Map<String, Boolean> waitingMap = new ConcurrentHashMap<>();
-
- public static void wait(String point) {
- if (KylinConfig.getInstanceFromEnv().isUTEnv()) {
- try {
- if (latchMap.containsKey(point)) {
- waitingMap.put(point, true);
- Thread.sleep(latchMap.get(point));
- latchMap.remove(point);
- waitingMap.remove(point);
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- public static void registerWaitPoint(String point, int ms) {
- Preconditions.checkState(!latchMap.containsKey(point));
- latchMap.put(point, ms);
- }
-
- public static boolean existWaitPoint(String point) {
- return latchMap.containsKey(point);
- }
-
- public static boolean isWaiting(String point) {
- return waitingMap.containsKey(point);
- }
-
- private SecondStorageConcurrentTestUtil() {
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageConfigLoader.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageConfigLoader.java
deleted file mode 100644
index c812621f4d..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageConfigLoader.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage;
-
-import io.kyligence.kap.secondstorage.config.ClusterInfo;
-
-public interface SecondStorageConfigLoader {
-
- void load();
-
- void refresh();
-
- ClusterInfo getCluster();
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageConstants.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageConstants.java
deleted file mode 100644
index c4a1415f30..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageConstants.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage;
-
-import java.util.Set;
-
-import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
-
-// CALL FROM CORE
-public class SecondStorageConstants {
- public static final String P_OLD_SEGMENT_IDS = "oldSegmentIds";
- public static final String P_MERGED_SEGMENT_ID = "mergedSegmentId";
- public static final String NODE_REPLICA = "kylin.second-storage.node-replica";
-
- // config
- public static final String CONFIG_SECOND_STORAGE_CLUSTER = "kylin.second-storage.cluster-config";
-
- // job
- public static final String STEP_EXPORT_TO_SECOND_STORAGE = "STEP_EXPORT_TO_SECOND_STORAGE";
- public static final String STEP_REFRESH_SECOND_STORAGE = "STEP_REFRESH_SECOND_STORAGE";
- public static final String STEP_MERGE_SECOND_STORAGE = "STEP_MERGE_SECOND_STORAGE";
-
- public static final String STEP_SECOND_STORAGE_NODE_CLEAN = "STEP_SECOND_STORAGE_NODE_CLEAN";
- public static final String STEP_SECOND_STORAGE_MODEL_CLEAN = "STEP_SECOND_STORAGE_MODEL_CLEAN";
- public static final String STEP_SECOND_STORAGE_SEGMENT_CLEAN = "STEP_SECOND_STORAGE_SEGMENT_CLEAN";
- public static final String STEP_SECOND_STORAGE_INDEX_CLEAN = "STEP_SECOND_STORAGE_INDEX_CLEAN";
- public static final String STEP_SECOND_STORAGE_REFRESH_SECONDARY_INDEX = "STEP_SECOND_STORAGE_REFRESH_SECONDARY_INDEX";
-
- public static final Set<String> SKIP_STEP_RUNNING = ImmutableSet.of(STEP_EXPORT_TO_SECOND_STORAGE,
- STEP_REFRESH_SECOND_STORAGE, STEP_MERGE_SECOND_STORAGE, STEP_SECOND_STORAGE_NODE_CLEAN,
- STEP_SECOND_STORAGE_MODEL_CLEAN, STEP_SECOND_STORAGE_INDEX_CLEAN,
- STEP_SECOND_STORAGE_REFRESH_SECONDARY_INDEX);
- public static final Set<String> SKIP_JOB_RUNNING = ImmutableSet.of(STEP_SECOND_STORAGE_SEGMENT_CLEAN);
-
- // internal config
- public static final String PROJECT_MODEL_SEGMENT_PARAM = "projectModelSegmentParam";
- public static final String PROJECT = "project";
-
-
-
- private SecondStorageConstants() {
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageLockUtils.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageLockUtils.java
deleted file mode 100644
index b92d6bb320..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageLockUtils.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage;
-
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.metadata.model.SegmentRange;
-
-public class SecondStorageLockUtils {
- private static final Map<Pair<String, SegmentRange<Long>>, Lock> JOB_LOCKS = new ConcurrentHashMap<>();
- private static final Object guard = new Object();
-
- public static boolean containsKey(String modelId) {
- return JOB_LOCKS.keySet().stream().anyMatch(item -> item.getFirst().equals(modelId));
- }
-
- public static boolean containsKey(String modelId, SegmentRange<Long> range) {
- return JOB_LOCKS.keySet().stream()
- .anyMatch(item -> item.getFirst().equals(modelId) && item.getSecond().overlaps(range));
- }
-
- private static Optional<Pair<String, SegmentRange<Long>>> getOverlapKey(String modelId, SegmentRange<Long> range) {
- return JOB_LOCKS.keySet().stream()
- .filter(item -> item.getFirst().equals(modelId) && item.getSecond().overlaps(range)).findFirst();
- }
-
- public static Lock acquireLock(String modelId, SegmentRange<Long> range) {
- Preconditions.checkNotNull(modelId);
- Preconditions.checkNotNull(range);
- int second = KylinConfig.getInstanceFromEnv().getSecondStorageWaitLockTimeout();
- synchronized (guard) {
- while (containsKey(modelId, range)) {
- Optional<Pair<String, SegmentRange<Long>>> key = getOverlapKey(modelId, range);
- if (key.isPresent()) {
- Lock lock = JOB_LOCKS.get(key.get());
- try {
- if (lock.tryLock(second, TimeUnit.SECONDS)) {
- lock.unlock();
- JOB_LOCKS.remove(key.get());
- } else {
- break;
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
- }
- if (containsKey(modelId, range)) {
- throw new IllegalStateException("Can't acquire job lock, job is running now.");
- }
- Preconditions.checkArgument(!containsKey(modelId, range));
- Lock lock = new ReentrantLock();
- JOB_LOCKS.put(new Pair<>(modelId, range), lock);
- return lock;
- }
- }
-
- public static void unlock(String modelId, SegmentRange<Long> range) {
- Pair<String, SegmentRange<Long>> key = new Pair<>(modelId, range);
- if (!JOB_LOCKS.containsKey(key)) {
- throw new IllegalStateException(String.format(Locale.ROOT,
- "Logical Error! This is a bug. Lock for model %s:%s is lost", modelId, range));
- }
- JOB_LOCKS.get(key).unlock();
- JOB_LOCKS.remove(key);
- }
-
- private SecondStorageLockUtils() {
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageNodeHelper.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageNodeHelper.java
deleted file mode 100644
index 0e229cd1e8..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageNodeHelper.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.QueryContext;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
-import io.kyligence.kap.secondstorage.config.ClusterInfo;
-import io.kyligence.kap.secondstorage.config.Node;
-import io.kyligence.kap.secondstorage.metadata.Manager;
-import io.kyligence.kap.secondstorage.metadata.NodeGroup;
-
-// CALL FROM CORE
-public class SecondStorageNodeHelper {
- private static final AtomicBoolean initialized = new AtomicBoolean(false);
- private static final ConcurrentMap<String, Node> NODE_MAP = new ConcurrentHashMap<>();
- private static final ConcurrentMap<String, String> NODE2PAIR_INDEX = new ConcurrentHashMap<>();
- private static final Map<String, List<Node>> CLUSTER = new ConcurrentHashMap<>();
- private static Function<Node, String> node2url;
- private static BiFunction<List<Node>, QueryContext, String> shard2url;
-
- public static void initFromCluster(ClusterInfo cluster, Function<Node, String> node2url,
- BiFunction<List<Node>, QueryContext, String> shard2url) {
- synchronized (SecondStorageNodeHelper.class) {
- NODE2PAIR_INDEX.clear();
- cluster.getNodes().forEach(node -> NODE_MAP.put(node.getName(), node));
- CLUSTER.putAll(cluster.getCluster());
- SecondStorageNodeHelper.node2url = node2url;
- SecondStorageNodeHelper.shard2url = shard2url;
-
- // build lookup table for node to pair
- CLUSTER.forEach((pair, nodes) -> {
- for (Node node : nodes) {
- NODE2PAIR_INDEX.put(node.getName(), pair);
- }
- });
- initialized.set(true);
- }
- }
-
- public static List<String> resolve(List<String> names) {
- Preconditions.checkState(initialized.get());
- return names.stream().map(name -> {
- Preconditions.checkState(NODE_MAP.containsKey(name));
- return node2url.apply(NODE_MAP.get(name));
- }).collect(Collectors.toList());
- }
-
- public static String getPairByNode(String node) {
- Preconditions.checkArgument(NODE2PAIR_INDEX.containsKey(node), "Node %s doesn't exist", node);
- return NODE2PAIR_INDEX.get(node);
- }
-
- public static List<String> resolveToJDBC(List<String> names) {
- return resolve(names);
- }
-
- public static List<String> resolveShardToJDBC(List<Set<String>> shardNames, QueryContext queryContext) {
- Preconditions.checkState(initialized.get());
- return shardNames.stream().map(replicaNames -> {
- List<Node> replicas = replicaNames.stream().map(replicaName -> {
- Preconditions.checkState(NODE_MAP.containsKey(replicaName));
- return NODE_MAP.get(replicaName);
- }).collect(Collectors.toList());
- Collections.shuffle(replicas);
- return shard2url.apply(replicas, queryContext);
- }).collect(Collectors.toList());
- }
-
- public static String resolve(String name) {
- Preconditions.checkState(initialized.get());
- return node2url.apply(NODE_MAP.get(name));
- }
-
- public static List<String> getPair(String pairName) {
- return CLUSTER.get(pairName).stream().map(Node::getName).collect(Collectors.toList());
- }
-
- private SecondStorageNodeHelper() {
- }
-
- public static Node getNode(String name) {
- return new Node(NODE_MAP.get(name));
- }
-
- public static List<Node> getALlNodes() {
- Preconditions.checkState(initialized.get());
- return new ArrayList<>(NODE_MAP.values());
- }
-
- public static List<Node> getALlNodesInProject(String project) {
- Optional<Manager<NodeGroup>> nodeGroupOptional = SecondStorageUtil
- .nodeGroupManager(KylinConfig.getInstanceFromEnv(), project);
- Preconditions.checkState(nodeGroupOptional.isPresent(), "node group manager is not init");
- return nodeGroupOptional.get().listAll().stream().flatMap(nodeGroup -> nodeGroup.getNodeNames().stream())
- .map(NODE_MAP::get).distinct().collect(Collectors.toList());
- }
-
- public static List<String> getAllNames() {
- Preconditions.checkState(initialized.get());
- return new ArrayList<>(NODE_MAP.keySet());
- }
-
- public static List<String> getAllPairs() {
- Preconditions.checkState(initialized.get());
- return new ArrayList<>(CLUSTER.keySet());
- }
-
- public static Map<Integer, List<String>> separateReplicaGroup(int replicaNum, String... pairs) {
- for (final String pair : pairs) {
- Preconditions.checkArgument(getPair(pair).size() == replicaNum,
- "Pair size is different from replica number");
- }
- Map<Integer, List<String>> nodeMap = new HashMap<>(replicaNum);
- for (final String pair : pairs) {
- List<String> pairNodes = getPair(pair);
- pairNodes.sort(String::compareTo);
- ListIterator<String> it = pairNodes.listIterator();
- while (it.hasNext()) {
- List<String> nodes = nodeMap.computeIfAbsent(it.nextIndex() % replicaNum, idx -> new ArrayList<>());
- nodes.add(it.next());
- }
- }
-
- return nodeMap;
- }
-
- /**
- * groups to shard
- * group [replica][shardSize] to shard[shardSize][replica]
- *
- * @param groups group
- * @return shards
- */
- public static List<Set<String>> groupsToShards(List<NodeGroup> groups) {
- int shardSize = groups.get(0).getNodeNames().size();
- // key is shard num, value is replica name
- Map<Integer, Set<String>> shards = new HashMap<>(shardSize);
-
- for (int shardNum = 0; shardNum < shardSize; shardNum++) {
- for (NodeGroup group : groups) {
- shards.computeIfAbsent(shardNum, key -> new HashSet<>()).add(group.getNodeNames().get(shardNum));
- }
- }
-
- return new ArrayList<>(shards.values());
- }
-
- public static void clear() {
- synchronized (SecondStorageNodeHelper.class) {
- initialized.set(false);
- NODE_MAP.clear();
- CLUSTER.clear();
- }
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStoragePlugin.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStoragePlugin.java
deleted file mode 100644
index 5e0bb2aaf6..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStoragePlugin.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage;
-
-import org.apache.kylin.common.KylinConfig;
-
-import io.kyligence.kap.secondstorage.metadata.Manager;
-import io.kyligence.kap.secondstorage.metadata.NodeGroup;
-import io.kyligence.kap.secondstorage.metadata.TableFlow;
-import io.kyligence.kap.secondstorage.metadata.TablePlan;
-
-public interface SecondStoragePlugin {
- boolean ready();
-
- String queryCatalog();
-
- Manager<TableFlow> tableFlowManager(KylinConfig config, String project);
-
- Manager<TablePlan> tablePlanManager(KylinConfig config, String project);
-
- Manager<NodeGroup> nodeGroupManager(KylinConfig config, String project);
-
- SecondStorageConfigLoader getConfigLoader();
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageQueryRouteUtil.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageQueryRouteUtil.java
deleted file mode 100644
index 95d2d2b623..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageQueryRouteUtil.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.QueryContext;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
-import io.kyligence.kap.secondstorage.metadata.NodeGroup;
-import io.kyligence.kap.secondstorage.metadata.TablePartition;
-
-public class SecondStorageQueryRouteUtil {
- private static final Map<String, Boolean> NODE_STATUS = new ConcurrentHashMap<>();
-
- private SecondStorageQueryRouteUtil() {
- }
-
- public static void setNodeStatus(String node, boolean status) {
- NODE_STATUS.put(node, status);
- }
-
- public static boolean getNodeStatus(String node) {
- return NODE_STATUS.getOrDefault(node, true);
- }
-
- public static List<Set<String>> getUsedShard(List<TablePartition> partitions, String project,
- Set<String> allSegIds) {
- // collect all node which partition used
- Set<String> allSegmentUsedNode = Sets.newHashSet();
- for (TablePartition partition : partitions) {
- if (allSegIds.contains(partition.getSegmentId())) {
- allSegmentUsedNode.addAll(partition.getShardNodes());
- }
- }
-
- if (allSegmentUsedNode.isEmpty()) {
- QueryContext.current().setRetrySecondStorage(false);
- throw new IllegalStateException("Segment node is empty.");
- }
- List<NodeGroup> nodeGroups = SecondStorage.nodeGroupManager(KylinConfig.getInstanceFromEnv(), project)
- .listAll();
-
- if (nodeGroups.isEmpty()) {
- QueryContext.current().setRetrySecondStorage(false);
- throw new IllegalStateException("Node groups is empty.");
- }
-
- List<Set<String>> shards = SecondStorageNodeHelper.groupsToShards(nodeGroups);
- List<Set<String>> segmentUsedShard = getSegmentUsedShard(shards, allSegmentUsedNode);
- filterAvailableReplica(segmentUsedShard);
-
- return segmentUsedShard;
- }
-
- /**
- * Get segment used nodes
- *
- * @param shards shards
- * @param allSegmentUsedNode all segment used node
- * @return segment used nodes
- */
- private static List<Set<String>> getSegmentUsedShard(List<Set<String>> shards, Set<String> allSegmentUsedNode) {
- // filter which shards used by partitions
- return shards.stream().filter(replicas -> {
- for (String nodeName : allSegmentUsedNode) {
- if (replicas.contains(nodeName)) {
- return true;
- }
- }
- return false;
- }).collect(Collectors.toList());
- }
-
- /**
- * filter available replica
- *
- * @param segmentUsedShard segments used shard
- */
- private static void filterAvailableReplica(List<Set<String>> segmentUsedShard) {
- for (Set<String> replicas : segmentUsedShard) {
- replicas.removeIf(replica -> !SecondStorageQueryRouteUtil.getNodeStatus(replica));
-
- if (replicas.isEmpty()) {
- QueryContext.current().setRetrySecondStorage(false);
- throw new IllegalStateException("One shard all replica has down");
- }
- }
- }
-
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageUpdater.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageUpdater.java
deleted file mode 100644
index cab92ad613..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageUpdater.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage;
-
-import java.util.Map;
-import java.util.Set;
-
-// CALL FROM CORE
-public interface SecondStorageUpdater {
- /**
- * Clean second storage model
- * if needed, return clean job id
- * @param project project
- * @param modelId model id
- * @return clean job id
- */
- String cleanModel(final String project, final String modelId);
-
- /**
- * disable model
- * @param project project
- * @param modelId model id
- * @return clean job id
- */
- String disableModel(final String project, final String modelId);
-
- /**
- * Update model index
- * if needed, return clean index id
- * @param project project
- * @param modelId model id
- * @return clean index job id
- */
- String updateIndex(String project, String modelId);
-
- /**
- * Get query metric , like scan rows / scan bytes
- * @param project project
- * @param queryId query id
- * @return metrics
- */
- Map<String, Object> getQueryMetric(String project, String queryId);
-
- String removeIndexByLayoutId(String project, String modelId, Set<Long> layoutIds);
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageUtil.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageUtil.java
deleted file mode 100644
index 5b92f57d36..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/SecondStorageUtil.java
+++ /dev/null
@@ -1,665 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage;
-
-import static org.apache.kylin.common.exception.ServerErrorCode.BASE_TABLE_INDEX_NOT_AVAILABLE;
-import static org.apache.kylin.common.exception.ServerErrorCode.PARTITION_COLUMN_NOT_AVAILABLE;
-import static org.apache.kylin.common.exception.ServerErrorCode.SECOND_STORAGE_DATA_NOT_EXIST;
-import static org.apache.kylin.common.exception.ServerErrorCode.SECOND_STORAGE_PROJECT_STATUS_ERROR;
-import static org.apache.kylin.job.constant.ExecutableConstants.STEP_NAME_BUILD_SPARK_CUBE;
-import static org.apache.kylin.job.constant.ExecutableConstants.STEP_NAME_CLEANUP;
-import static org.apache.kylin.job.constant.ExecutableConstants.STEP_NAME_MERGER_SPARK_SEGMENT;
-import static org.apache.kylin.job.constant.ExecutableConstants.STEP_UPDATE_METADATA;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.exception.JobErrorCode;
-import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.exception.ServerErrorCode;
-import org.apache.kylin.common.msg.MsgPicker;
-import org.apache.kylin.common.persistence.transaction.UnitOfWork;
-import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.dao.ExecutablePO;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ChainedExecutable;
-import org.apache.kylin.job.execution.ChainedStageExecutable;
-import org.apache.kylin.job.execution.DefaultExecutable;
-import org.apache.kylin.job.execution.ExecutableManager;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.execution.JobSchedulerModeEnum;
-import org.apache.kylin.job.execution.JobTypeEnum;
-import org.apache.kylin.job.execution.StageBase;
-import org.apache.kylin.job.util.JobContextUtil;
-import org.apache.kylin.metadata.cube.model.IndexEntity;
-import org.apache.kylin.metadata.cube.model.IndexPlan;
-import org.apache.kylin.metadata.cube.model.LayoutEntity;
-import org.apache.kylin.metadata.cube.model.NDataSegment;
-import org.apache.kylin.metadata.cube.model.NDataflow;
-import org.apache.kylin.metadata.cube.model.NDataflowManager;
-import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
-import org.apache.kylin.metadata.model.NDataModel;
-import org.apache.kylin.metadata.model.NDataModelManager;
-import org.apache.kylin.metadata.model.SegmentRange;
-import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
-import org.apache.kylin.metadata.query.NativeQueryRealization;
-
-import io.kyligence.kap.secondstorage.config.Node;
-import io.kyligence.kap.secondstorage.enums.LockTypeEnum;
-import io.kyligence.kap.secondstorage.metadata.Manager;
-import io.kyligence.kap.secondstorage.metadata.NodeGroup;
-import io.kyligence.kap.secondstorage.metadata.TableData;
-import io.kyligence.kap.secondstorage.metadata.TableFlow;
-import io.kyligence.kap.secondstorage.metadata.TablePartition;
-import io.kyligence.kap.secondstorage.metadata.TablePlan;
-import io.kyligence.kap.secondstorage.response.SecondStorageInfo;
-import io.kyligence.kap.secondstorage.response.SecondStorageNode;
-
-// CALL FROM CORE
-public class SecondStorageUtil {
- public static final Set<ExecutableState> RUNNING_STATE = ImmutableSet.of(ExecutableState.RUNNING,
- ExecutableState.READY, ExecutableState.PAUSED, ExecutableState.PENDING);
- public static final Set<JobTypeEnum> RELATED_JOBS = ImmutableSet.of(JobTypeEnum.INDEX_BUILD,
- JobTypeEnum.INDEX_REFRESH, JobTypeEnum.INC_BUILD, JobTypeEnum.INDEX_MERGE,
- JobTypeEnum.EXPORT_TO_SECOND_STORAGE, JobTypeEnum.SECOND_STORAGE_REFRESH_SECONDARY_INDEXES);
- public static final Set<JobTypeEnum> BUILD_JOBS = ImmutableSet.of(JobTypeEnum.INDEX_BUILD,
- JobTypeEnum.INDEX_REFRESH, JobTypeEnum.INC_BUILD, JobTypeEnum.INDEX_MERGE);
- public static final Set<String> EXPORT_STEPS = ImmutableSet.of(SecondStorageConstants.STEP_EXPORT_TO_SECOND_STORAGE,
- SecondStorageConstants.STEP_REFRESH_SECOND_STORAGE, SecondStorageConstants.STEP_MERGE_SECOND_STORAGE);
-
- public static final Pattern PUSHED_AGGREGATES = Pattern.compile("PushedAggregates: \\[[^\\]]++\\]");
- public static final Pattern PUSHED_GROUP_BY = Pattern.compile("PushedGroupByExpressions: \\[[^\\]]++\\]");
- public static final Pattern PUSHED_FILTERS = Pattern.compile("PushedFilters: \\[[^\\]]++\\]");
- public static final Pattern PUSHED_LIMIT = Pattern.compile("PushedLimit: LIMIT [0-9]+");
- public static final Pattern PUSHED_OFFSET = Pattern.compile("PushedOffset: OFFSET [0-9]+");
- public static final Pattern PUSHED_TOP_N = Pattern.compile("PushedTopN: ORDER BY \\[.+?\\] LIMIT [0-9]+");
-
- private SecondStorageUtil() {
- }
-
- public static void initModelMetaData(String project, String model) {
- checkEnableModel(project, model);
- EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- final KylinConfig config = KylinConfig.getInstanceFromEnv();
- Optional<Manager<TablePlan>> tablePlanManager = tablePlanManager(config, project);
- Optional<Manager<TableFlow>> tableFlowManager = tableFlowManager(config, project);
- Preconditions.checkState(tableFlowManager.isPresent() && tablePlanManager.isPresent());
- NIndexPlanManager indexPlanManager = NIndexPlanManager.getInstance(config, project);
- TablePlan tablePlan = tablePlanManager.get().makeSureRootEntity(model);
- tableFlowManager.get().makeSureRootEntity(model);
- Map<Long, List<LayoutEntity>> layouts = indexPlanManager.getIndexPlan(model).getAllLayoutsMap().values()
- .stream().filter(SecondStorageUtil::isBaseTableIndex)
- .collect(Collectors.groupingBy(LayoutEntity::getIndexId));
- for (Map.Entry<Long, List<LayoutEntity>> entry : layouts.entrySet()) {
- LayoutEntity layoutEntity = entry.getValue().stream().filter(SecondStorageUtil::isBaseTableIndex)
- .findFirst().orElse(null);
- Preconditions.checkNotNull(layoutEntity);
- tablePlan = tablePlan.createTableEntityIfNotExists(layoutEntity, true);
- }
- return null;
- }, project, 1, UnitOfWork.DEFAULT_EPOCH_ID);
- }
-
- public static void checkSecondStorageData(String project) {
- if (!SecondStorageUtil.isProjectEnable(project)) {
- throw new KylinException(SECOND_STORAGE_PROJECT_STATUS_ERROR,
- String.format(Locale.ROOT, "'%s' not enable second storage.", project));
- }
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- Optional<Manager<TableFlow>> optionalTableFlowManager = SecondStorageUtil.tableFlowManager(config, project);
- if (!optionalTableFlowManager.isPresent()) {
- throw new KylinException(SECOND_STORAGE_DATA_NOT_EXIST,
- String.format(Locale.ROOT, "'%s' second storage data not exist.", project));
- }
- }
-
- private static void checkEnableModel(String project, String model) {
- NIndexPlanManager indexPlanManager = NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- final IndexPlan indexPlan = indexPlanManager.getIndexPlan(model);
- if (!indexPlan.containBaseTableLayout()) {
- throw new KylinException(BASE_TABLE_INDEX_NOT_AVAILABLE,
- MsgPicker.getMsg().getBaseTableIndexNotAvailable());
- }
- if (indexPlan.getModel().isIncrementBuildOnExpertMode()) {
- boolean containPartitionCol = indexPlan.getBaseTableLayout().getColumns().stream().anyMatch(col -> col
- .getTableDotName().equals(indexPlan.getModel().getPartitionDesc().getPartitionDateColumn()));
- if (!containPartitionCol) {
- throw new KylinException(PARTITION_COLUMN_NOT_AVAILABLE,
- MsgPicker.getMsg().getPartitionColumnNotAvailable());
- }
- }
- }
-
- public static boolean isBaseTableIndex(LayoutEntity index) {
- return index != null && IndexEntity.isTableIndex(index.getId()) && index.isBaseIndex();
- }
-
- public static LayoutEntity getBaseIndex(NDataflow df) {
- return df.getIndexPlan().getAllLayouts().stream().filter(SecondStorageUtil::isBaseTableIndex)
- .collect(Collectors.toList()).get(0);
- }
-
- public static List<String> getProjectLocks(String project) {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- Optional<Manager<NodeGroup>> nodeGroupManager = SecondStorageUtil.nodeGroupManager(config, project);
- if (!nodeGroupManager.isPresent()) {
- return new ArrayList<>();
- }
- List<NodeGroup> nodeGroups = nodeGroupManager.get().listAll();
- if (CollectionUtils.isNotEmpty(nodeGroups)) {
- Preconditions.checkState(nodeGroups.stream().map(NodeGroup::getLockTypes).distinct().count() == 1,
- "Logical Error, this is a bug! Cluster has different lock type.");
- return nodeGroups.get(0).getLockTypes();
- }
- return new ArrayList<>();
- }
-
- public static List<AbstractExecutable> findSecondStorageRelatedJobByProject(String project) {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- ExecutableManager executableManager = ExecutableManager.getInstance(config, project);
- return executableManager.getExecutablesByJobType(RELATED_JOBS);
- }
-
- public static void validateProjectLock(String project, List<String> requestLocks) {
- LockTypeEnum.checkLocks(requestLocks, SecondStorageUtil.getProjectLocks(project));
- }
-
- public static void validateDisableModel(String project, String modelId) {
- validateProjectLock(project, Arrays.asList(LockTypeEnum.LOAD.name()));
- List<AbstractExecutable> jobs = SecondStorageUtil.findSecondStorageRelatedJobByProject(project);
- if (jobs.stream().filter(job -> RUNNING_STATE.contains(job.getStatusInMem()))
- .anyMatch(job -> job.getTargetSubject().equals(modelId))) {
- String name = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), project)
- .getDataModelDesc(modelId).getAlias();
- throw new KylinException(JobErrorCode.SECOND_STORAGE_JOB_EXISTS,
- String.format(Locale.ROOT, MsgPicker.getMsg().getSecondStorageJobExists(), name));
- }
- }
-
- public static boolean isGlobalEnable() {
- return SecondStorage.enabled();
- }
-
- public static boolean isProjectEnable(String project) {
- if (isGlobalEnable()) {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- Optional<Manager<NodeGroup>> nodeGroupManager = nodeGroupManager(config, project);
- return nodeGroupManager.isPresent() && !nodeGroupManager.get().listAll().isEmpty();
- }
- return false;
- }
-
- public static List<SecondStorageNode> listProjectNodes(String project) {
- if (!isProjectEnable(project)) {
- return Collections.emptyList();
- }
- Optional<Manager<NodeGroup>> groupManager = nodeGroupManager(KylinConfig.getInstanceFromEnv(), project);
- Preconditions.checkNotNull(groupManager);
- return groupManager.map(nodeGroupNManager -> nodeGroupNManager.listAll().stream()
- .flatMap(nodeGroup -> nodeGroup.getNodeNames().stream()).distinct().map(name -> {
- Node node = SecondStorageNodeHelper.getNode(name);
- return new SecondStorageNode(node);
- }).collect(Collectors.toList())).orElse(Collections.emptyList());
- }
-
- public static boolean isModelEnable(String project, String model) {
- // if (isProjectEnable(project)) {
- // KylinConfig config = KylinConfig.getInstanceFromEnv();
- // Optional<Manager<TableFlow>> tableFlowManager = tableFlowManager(config, project);
- // return tableFlowManager.isPresent() && tableFlowManager.get().get(model).isPresent();
- // }
- return false;
- }
-
- public static List<SecondStorageInfo> setSecondStorageSizeInfo(List<NDataModel> models) {
- if (models == null || models.isEmpty()) {
- return Collections.emptyList();
- }
- Optional<Manager<TableFlow>> tableFlowManager = SecondStorageUtil
- .tableFlowManager(KylinConfig.getInstanceFromEnv(), models.get(0).getProject());
- Preconditions.checkState(tableFlowManager.isPresent());
- return setSecondStorageSizeInfo(models, tableFlowManager.get());
- }
-
- protected static List<SecondStorageInfo> setSecondStorageSizeInfo(List<NDataModel> models,
- Manager<TableFlow> tableFlowManager) {
- List<Set<String>> shards = SecondStorageNodeHelper.groupsToShards(
- SecondStorageUtil.listNodeGroup(KylinConfig.getInstanceFromEnv(), models.get(0).getProject()));
-
- return models.stream().map(model -> {
- SecondStorageInfo secondStorageInfo = new SecondStorageInfo();
- secondStorageInfo.setSecondStorageEnabled(isModelEnable(model.getProject(), model.getId()));
- TableFlow tableFlow = tableFlowManager.get(model.getId()).orElse(null);
- if (isTableFlowEmpty(tableFlow)) {
- secondStorageInfo.setSecondStorageNodes(Collections.emptyMap());
- secondStorageInfo.setSecondStorageSize(0);
- } else {
- List<TablePartition> tablePartitions = tableFlow.getTableDataList().stream()
- .flatMap(tableData -> tableData.getPartitions().stream()).collect(Collectors.toList());
- Set<String> nodesStr = Sets.newHashSet();
-
- for (TablePartition partition : tablePartitions) {
- nodesStr.addAll(partition.getShardNodes());
- }
-
- secondStorageInfo.setSecondStorageNodes(convertNodesToPairs(new ArrayList<>(nodesStr)));
- secondStorageInfo.setSecondStorageSize(calculateSecondStorageSize(shards, tablePartitions));
- }
- return secondStorageInfo;
- }).collect(Collectors.toList());
- }
-
- public static long calculateSecondStorageSize(List<Set<String>> shards, List<TablePartition> partitions) {
- // key: node_name / value: node_size
- Map<String, Long> nodesSize = partitions.stream()
- .flatMap(partition -> partition.getSizeInNode().entrySet().stream())
- .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, Long::sum));
-
- return shards.stream().mapToLong(
- shard -> shard.stream().mapToLong(replica -> nodesSize.getOrDefault(replica, 0L)).max().orElse(0))
- .sum();
- }
-
- public static SecondStorageNode transformNode(String name) {
- Node node = SecondStorageNodeHelper.getNode(name);
- return new SecondStorageNode(node);
- }
-
- public static Map<String, List<SecondStorageNode>> convertNodesToPairs(List<String> nodes) {
- Map<String, List<SecondStorageNode>> result = Maps.newHashMap();
- nodes.stream().sorted()
- .forEach(node -> result
- .computeIfAbsent(SecondStorageNodeHelper.getPairByNode(node), k -> new ArrayList<>())
- .add(new SecondStorageNode(SecondStorageNodeHelper.getNode(node))));
- return result;
- }
-
- public static boolean isTableFlowEmpty(TableFlow tableFlow) {
- return tableFlow == null || tableFlow.getTableDataList() == null || tableFlow.getTableDataList().isEmpty()
- || tableFlow.getTableDataList().get(0).getPartitions() == null
- || tableFlow.getTableDataList().get(0).getPartitions().isEmpty();
- }
-
- public static void disableProject(String project) {
- EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- Optional<Manager<NodeGroup>> nodeGroupManager = SecondStorageUtil.nodeGroupManager(config, project);
- Optional<Manager<TableFlow>> tableFlowManager = SecondStorageUtil.tableFlowManager(config, project);
- Optional<Manager<TablePlan>> tablePlanManager = SecondStorageUtil.tablePlanManager(config, project);
- nodeGroupManager.ifPresent(manager -> manager.listAll().forEach(manager::delete));
- tableFlowManager.ifPresent(manager -> manager.listAll().forEach(manager::delete));
- tablePlanManager.ifPresent(manager -> manager.listAll().forEach(manager::delete));
- return null;
- }, project, 1, UnitOfWork.DEFAULT_EPOCH_ID);
- }
-
- public static void disableModel(String project, String modelId) {
- EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- Optional<Manager<TableFlow>> tableFlowManager = SecondStorageUtil.tableFlowManager(config, project);
- Optional<Manager<TablePlan>> tablePlanManager = SecondStorageUtil.tablePlanManager(config, project);
- tablePlanManager.ifPresent(manager -> manager.listAll().stream()
- .filter(tablePlan -> tablePlan.getId().equals(modelId)).forEach(manager::delete));
- tableFlowManager.ifPresent(manager -> manager.listAll().stream()
- .filter(tableFlow -> tableFlow.getId().equals(modelId)).forEach(manager::delete));
- return null;
- }, project, 1, UnitOfWork.DEFAULT_EPOCH_ID);
- }
-
- public static void cleanSegments(String project, String model, Set<String> segments) {
- EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- Optional<Manager<TableFlow>> tableFlowManager = SecondStorageUtil.tableFlowManager(config, project);
- tableFlowManager.ifPresent(manager -> manager.listAll().stream()
- .filter(tableFlow -> tableFlow.getId().equals(model))
- .forEach(tableFlow -> tableFlow.update(copy -> copy.getTableDataList().stream()
- .filter(tableData -> tableData.getDatabase().equals(NameUtil.getDatabase(config, project))
- && tableData.getTable().startsWith(NameUtil.tablePrefix(model)))
- .forEach(tableData -> tableData.removePartitions(
- tablePartition -> segments.contains(tablePartition.getSegmentId()))))));
- return null;
- }, project, 1, UnitOfWork.DEFAULT_EPOCH_ID);
- }
-
- public static void cleanSegments(String project, String modelId, Set<String> segments, Set<Long> layoutIds) {
- EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- Optional<Manager<TableFlow>> tableFlowManager = SecondStorageUtil.tableFlowManager(config, project);
-
- tableFlowManager.ifPresent(
- manager -> manager.listAll().stream().filter(tableFlow -> tableFlow.getId().equals(modelId))
- .forEach(tableFlow -> tableFlow.update(copy -> copy.getTableDataList().stream()
- .filter(tableData -> layoutIds.contains(tableData.getLayoutID()))
- .forEach(tableData -> tableData.removePartitions(
- tablePartition -> segments.contains(tablePartition.getSegmentId()))))));
-
- return null;
- }, project, 1, UnitOfWork.DEFAULT_EPOCH_ID);
- }
-
- public static Optional<Manager<TableFlow>> tableFlowManager(KylinConfig config, String project) {
- return isGlobalEnable() ? Optional.of(SecondStorage.tableFlowManager(config, project)) : Optional.empty();
- }
-
- public static List<TableFlow> listTableFlow(KylinConfig config, String project) {
- Optional<Manager<TableFlow>> optional = tableFlowManager(config, project);
- if (optional.isPresent()) {
- Manager<TableFlow> tableFlowManager = optional.get();
- return tableFlowManager.listAll();
- }
- return Collections.emptyList();
- }
-
- public static TableFlow getTableFlow(String project, String modelId) {
- Optional<Manager<TableFlow>> tableFlowManager = tableFlowManager(KylinConfig.getInstanceFromEnv(), project);
- Preconditions.checkState(tableFlowManager.isPresent());
- Optional<TableFlow> tableFlow = tableFlowManager.get().get(modelId);
- Preconditions.checkState(tableFlow.isPresent());
- return tableFlow.get();
- }
-
- public static Optional<Manager<TableFlow>> tableFlowManager(NDataflow dataflow) {
- return isGlobalEnable() ? tableFlowManager(dataflow.getConfig(), dataflow.getProject()) : Optional.empty();
- }
-
- public static Optional<Manager<TablePlan>> tablePlanManager(KylinConfig config, String project) {
- return isGlobalEnable() ? Optional.of(SecondStorage.tablePlanManager(config, project)) : Optional.empty();
- }
-
- public static TablePlan getTablePlan(String project, String modelId) {
- Optional<Manager<TablePlan>> tablePlanManager = tablePlanManager(KylinConfig.getInstanceFromEnv(), project);
- Preconditions.checkState(tablePlanManager.isPresent());
- Optional<TablePlan> tablePlan = tablePlanManager.get().get(modelId);
- Preconditions.checkState(tablePlan.isPresent());
- return tablePlan.get();
- }
-
- public static Optional<Manager<NodeGroup>> nodeGroupManager(KylinConfig config, String project) {
- return isGlobalEnable() ? Optional.of(SecondStorage.nodeGroupManager(config, project)) : Optional.empty();
- }
-
- public static List<NodeGroup> listNodeGroup(KylinConfig config, String project) {
- Optional<Manager<NodeGroup>> optional = nodeGroupManager(config, project);
- if (optional.isPresent()) {
- Manager<NodeGroup> nodeGroupManager = optional.get();
- return nodeGroupManager.listAll();
- }
- return Collections.emptyList();
- }
-
- public static ExecutableState getJobStatus(String project, String jobId) {
- ExecutableManager manager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- return manager.getJob(jobId).getStatusInMem();
- }
-
- public static void checkJobRestart(String project, String jobId) {
- if (!isProjectEnable(project)) {
- return;
- }
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- ExecutablePO executablePO = JobContextUtil.getJobInfoDao(config).getExecutablePOByUuid(jobId);
- AbstractExecutable executable = ExecutableManager.getInstance(config, project).fromPO(executablePO);
- checkJobRestart(project, executable);
- }
-
- public static void checkJobRestart(String project, AbstractExecutable executable) {
- if (!isProjectEnable(project))
- return;
- JobTypeEnum type = executable.getJobType();
- boolean canRestart = type != JobTypeEnum.EXPORT_TO_SECOND_STORAGE
- && type != JobTypeEnum.SECOND_STORAGE_REFRESH_SECONDARY_INDEXES;
- if (BUILD_JOBS.contains(executable.getJobType()) && hasSecondStorageLoadJob(executable)) {
- if (executable.getJobSchedulerMode() == JobSchedulerModeEnum.DAG) {
- Optional<ChainedStageExecutable> detectResourceTask = getChainedStageExecutableByName(
- ExecutableConstants.STEP_NAME_DETECT_RESOURCE, executable);
- canRestart = detectResourceTask.filter(task -> task.getStatus() != ExecutableState.SUCCEED).isPresent();
- } else {
- long finishedCount = ((DefaultExecutable) executable).getTasks().stream()
- .filter(task -> ExecutableState.SUCCEED == task.getStatus()).count();
- canRestart = finishedCount < getExportStepPosition(executable) - 1;
- }
- }
- if (!canRestart) {
- throw new KylinException(ServerErrorCode.JOB_RESTART_FAILED, MsgPicker.getMsg().getJobRestartFailed());
- }
- }
-
- public static boolean checkStorageEmpty(String project, String modelId, long layoutId) {
- List<TableData> tableDataList = getTableFlow(project, modelId).getTableData(layoutId);
- return CollectionUtils.isEmpty(tableDataList)
- || tableDataList.stream().allMatch(tableData -> CollectionUtils.isEmpty(tableData.getPartitions()));
- }
-
- public static void checkSegmentRemove(String project, String modelId, String[] ids) {
- if (!isModelEnable(project, modelId))
- return;
- NDataflowManager dataflowManager = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- NDataflow dataflow = dataflowManager.getDataflow(modelId);
- long startTime = Long.MAX_VALUE - 1;
- long endTime = 0L;
- for (NDataSegment segment : dataflow.getSegments(Sets.newHashSet(ids))) {
- startTime = Math.min(startTime, Long.parseLong(segment.getSegRange().getStart().toString()));
- endTime = endTime > Long.parseLong(segment.getSegRange().getStart().toString()) ? endTime
- : Long.parseLong(segment.getSegRange().getEnd().toString());
- }
- if (SecondStorageLockUtils.containsKey(modelId,
- new SegmentRange.TimePartitionedSegmentRange(startTime, endTime))) {
- throw new KylinException(ServerErrorCode.SEGMENT_DROP_FAILED, MsgPicker.getMsg().getSegmentDropFailed());
- }
- }
-
- public static void checkJobResume(String project, String jobId) {
- if (!isProjectEnable(project))
- return;
- checkJobResume(ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv(), project).getJob(jobId));
- }
-
- @VisibleForTesting
- public static void checkJobResume(AbstractExecutable executable) {
- long runningJobsCount = ((DefaultExecutable) executable).getTasks().stream()
- .filter(task -> EXPORT_STEPS.contains(task.getName()))
- .filter(task -> ExecutableState.RUNNING == task.getStatus()).count();
- if (runningJobsCount != 0) {
- throw new KylinException(ServerErrorCode.JOB_RESUME_FAILED, MsgPicker.getMsg().getJobResumeFailed());
- }
- }
-
- public static void checkJobPause(String project, String jobId) {
- if (!isProjectEnable(project)) {
- return;
- }
- checkJobPause(ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv(), project).getJob(jobId));
- }
-
- private static void checkJobPause(AbstractExecutable executable) {
- if (executable.getJobType() == JobTypeEnum.SECOND_STORAGE_REFRESH_SECONDARY_INDEXES) {
- throw new KylinException(ServerErrorCode.JOB_PAUSE_FAILED, MsgPicker.getMsg().getJobPauseFailed());
- }
- }
-
- private static boolean hasSecondStorageLoadJob(AbstractExecutable executable) {
- return ((DefaultExecutable) executable).getTasks().stream()
- .anyMatch(task -> EXPORT_STEPS.contains(task.getName()));
- }
-
- private static int getExportStepPosition(AbstractExecutable executable) {
- int exportStepPosition = 0;
- for (AbstractExecutable task : ((DefaultExecutable) executable).getTasks()) {
- exportStepPosition++;
- if (EXPORT_STEPS.contains(task.getName())) {
- break;
- }
- }
- return exportStepPosition;
- }
-
- public static boolean checkBuildFlatTableIsSuccess(AbstractExecutable executable) {
- if (executable.getJobSchedulerMode() != JobSchedulerModeEnum.DAG) {
- return true;
- }
- Optional<ChainedStageExecutable> buildSparkCubeTask = getChainedStageExecutableByName(
- STEP_NAME_BUILD_SPARK_CUBE, executable);
- if (!buildSparkCubeTask.isPresent()) {
- return true;
- }
- Map<String, List<StageBase>> stages = buildSparkCubeTask.get().getStagesMap();
- return stages.entrySet().stream().map(segmentStages -> {
- String segmentId = segmentStages.getKey();
- return segmentStages.getValue().stream()
- .filter(segmentStage -> ExecutableConstants.STAGE_NAME_GENERATE_FLAT_TABLE
- .equals(segmentStage.getName()))
- .map(segmentStage -> segmentStage.getOutput(segmentId).getState() == ExecutableState.SUCCEED)
- .findFirst().orElse(true);
- }).reduce((b1, b2) -> b1 && b2).orElse(true);
- }
-
- private static Optional<ChainedStageExecutable> getChainedStageExecutableByName(String name,
- AbstractExecutable executable) {
- List<? extends AbstractExecutable> tasks = ((ChainedExecutable) executable).getTasks();
- return tasks.stream().filter(task -> name.equals(task.getName())).map(ChainedStageExecutable.class::cast)
- .findFirst();
- }
-
- public static boolean checkMergeFlatTableIsSuccess(AbstractExecutable executable) {
- if (executable.getJobSchedulerMode() != JobSchedulerModeEnum.DAG) {
- return true;
- }
- Optional<ChainedStageExecutable> mergeSparkCubeTask = getChainedStageExecutableByName(
- STEP_NAME_MERGER_SPARK_SEGMENT, executable);
- if (!mergeSparkCubeTask.isPresent()) {
- return true;
- }
- Map<String, List<StageBase>> stages = mergeSparkCubeTask.get().getStagesMap();
- return stages.entrySet().stream().map(segmentStages -> {
- String segmentId = segmentStages.getKey();
- return segmentStages.getValue().stream()
- .filter(segmentStage -> ExecutableConstants.STAGE_NAME_MERGE_FLAT_TABLE
- .equals(segmentStage.getName()))
- .map(segmentStage -> segmentStage.getOutput(segmentId).getState() == ExecutableState.SUCCEED)
- .findFirst().orElse(true);
- }).reduce((b1, b2) -> b1 && b2).orElse(true);
- }
-
- public static boolean checkBuildDfsIsSuccess(AbstractExecutable executable) {
- if (executable.getJobSchedulerMode() != JobSchedulerModeEnum.DAG) {
- return true;
- }
- List<? extends AbstractExecutable> tasks = ((ChainedExecutable) executable).getTasks();
-
- return tasks.stream().filter(task -> STEP_UPDATE_METADATA.equals(task.getName()))
- .map(task -> task.getStatus() == ExecutableState.SUCCEED).findFirst().orElse(true);
- }
-
- public static boolean checkMergeDfsIsSuccess(AbstractExecutable executable) {
- if (executable.getJobSchedulerMode() != JobSchedulerModeEnum.DAG) {
- return true;
- }
-
- List<? extends AbstractExecutable> tasks = ((ChainedExecutable) executable).getTasks();
-
- return tasks.stream().filter(task -> STEP_NAME_CLEANUP.equals(task.getName()))
- .map(task -> task.getStatus() == ExecutableState.SUCCEED).findFirst().orElse(true);
- }
-
- public static Set<Long> listEnableLayoutBySegment(String project, String modelId, String segmentId) {
- if (!isModelEnable(project, modelId)) {
- return ImmutableSet.of();
- }
-
- Optional<Manager<TableFlow>> tableFlowManager = tableFlowManager(KylinConfig.getInstanceFromEnv(), project);
-
- if (!tableFlowManager.isPresent()) {
- return ImmutableSet.of();
- }
-
- Optional<TableFlow> tableFlow = tableFlowManager.get().get(modelId);
-
- if (!tableFlow.isPresent()) {
- return ImmutableSet.of();
- }
- NDataflowManager dataflowManager = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
- NDataflow dataflow = dataflowManager.getDataflow(modelId);
- return tableFlow.get().getLayoutBySegment(segmentId).stream().filter(
- layout -> dataflow.getIndexPlan() != null && dataflow.getIndexPlan().getLayoutEntity(layout) != null)
- .collect(Collectors.toSet());
- }
-
- public static String collectExecutedPlan(String executedPlan) {
- StringBuilder pushedExecutedPlan = new StringBuilder();
- List<Pattern> pushedPattern = Lists.newArrayList(PUSHED_AGGREGATES, PUSHED_FILTERS, PUSHED_GROUP_BY,
- PUSHED_LIMIT, PUSHED_OFFSET, PUSHED_TOP_N);
- for (Pattern pattern : pushedPattern) {
- Matcher match = pattern.matcher(executedPlan);
- if (match.find()) {
- pushedExecutedPlan.append(",").append(match.group());
- }
- }
- return pushedExecutedPlan.toString();
- }
-
- public static String convertExecutedPlan(String executedPlan, String project,
- List<NativeQueryRealization> realizations) {
- Pattern columnName = Pattern.compile("c[0-9]+");
- List<NativeQueryRealization> secondStorageLayout = realizations.stream()
- .filter(NativeQueryRealization::isSecondStorage).collect(Collectors.toList());
- if (secondStorageLayout.isEmpty() || project == null)
- return null;
-
- NativeQueryRealization realization = secondStorageLayout.get(0);
- IndexPlan indexPlan = NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(), project)
- .getIndexPlan(realization.getModelId());
- LayoutEntity layout = indexPlan.getLayoutEntity(realization.getLayoutId());
-
- Matcher match = columnName.matcher(executedPlan);
- while (match.find()) {
- String name = match.group(0);
- Integer colIndex = convertColumnName(name);
- String alias = layout.getOrderedDimensions().get(colIndex).getAliasDotName();
- executedPlan = executedPlan.replace(match.group(), alias);
- match = columnName.matcher(executedPlan);
- }
- return executedPlan;
- }
-
- private static Integer convertColumnName(String name) {
- Pattern columnName = Pattern.compile("[0-9]+");
- Matcher match = columnName.matcher(name);
- if (match.find()) {
- return Integer.parseInt(match.group());
- }
- return null;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/ClusterInfo.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/ClusterInfo.java
deleted file mode 100644
index c7f6a258f3..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/ClusterInfo.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.config;
-
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.stream.Collectors;
-
-import org.apache.kylin.common.util.EncryptUtil;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.guava30.shaded.common.collect.Maps;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
-public class ClusterInfo {
- private Map<String, List<Node>> cluster;
- private String socketTimeout;
- private String keepAliveTimeout;
- private String connectTimeout;
- private String installPath;
- private String logPath;
-
- //username of machine
- private String userName;
- private String password;
- private String extConfig;
-
- @JsonIgnore
- public List<Node> getNodes() {
- return Collections.unmodifiableList(cluster.values().stream() //
- .flatMap(List::stream).collect(Collectors.toList()));
- }
-
- public ClusterInfo setCluster(Map<String, List<Node>> cluster) {
- this.cluster = cluster;
- return this;
- }
-
- public Map<String, List<Node>> getCluster() {
- TreeMap<String, List<Node>> orderedMap = new TreeMap<>(cluster);
- return Collections.unmodifiableMap(orderedMap);
- }
-
- public void transformNode() {
- for (final Map.Entry<String, List<Node>> pair : cluster.entrySet()) {
- List<Node> nodes = pair.getValue();
- List<Node> transformedNodes = new ArrayList<>();
- for (Object node : nodes) {
- Node n;
- if (!(node instanceof Node)) {
- n = JsonUtil.readValueQuietly(
- JsonUtil.writeValueAsStringQuietly(node).getBytes(StandardCharsets.UTF_8), Node.class);
- } else {
- n = (Node) node;
- }
- if (n.getSSHPort() == 0) {
- n.setSSHPort(22);
- }
- transformedNodes.add(n);
- }
- cluster.put(pair.getKey(), transformedNodes);
- }
- }
-
- public String getSocketTimeout() {
- return socketTimeout;
- }
-
- public ClusterInfo setSocketTimeout(String socketTimeout) {
- this.socketTimeout = socketTimeout;
- return this;
- }
-
- public String getKeepAliveTimeout() {
- return keepAliveTimeout;
- }
-
- public String getConnectTimeout() {
- return connectTimeout;
- }
-
- public ClusterInfo setConnectTimeout(String connectTimeout) {
- this.connectTimeout = connectTimeout;
- return this;
- }
-
- public String getExtConfig() {
- return extConfig;
- }
-
- public ClusterInfo setExtConfig(String extConfig) {
- this.extConfig = extConfig;
- return this;
- }
-
- public ClusterInfo setKeepAliveTimeout(String keepAliveTimeout) {
- this.keepAliveTimeout = keepAliveTimeout;
- return this;
- }
-
- public String getInstallPath() {
- return installPath;
- }
-
- public ClusterInfo setInstallPath(String installPath) {
- this.installPath = installPath;
- return this;
- }
-
- public String getLogPath() {
- return logPath;
- }
-
- public ClusterInfo setLogPath(String logPath) {
- this.logPath = logPath;
- return this;
- }
-
- public String getUserName() {
- return userName;
- }
-
- public void setUserName(String userName) {
- this.userName = userName;
- }
-
- public String getPassword() {
- return EncryptUtil.isEncrypted(password) ? EncryptUtil.decryptPassInKylin(password) : password;
- }
-
- public void setPassword(String password) {
- this.password = password;
- }
-
- public ClusterInfo(ClusterInfo cluster) {
- this.cluster = Maps.newHashMap(cluster.getCluster());
- this.keepAliveTimeout = cluster.getKeepAliveTimeout();
- this.socketTimeout = cluster.getSocketTimeout();
- this.connectTimeout = cluster.getConnectTimeout();
- this.logPath = cluster.getLogPath();
- this.userName = cluster.getUserName();
- this.password = cluster.getPassword();
- this.installPath = cluster.getInstallPath();
- this.extConfig = cluster.getExtConfig();
- }
-
- public boolean emptyCluster() {
- return cluster == null || cluster.isEmpty();
- }
-
- public ClusterInfo() {
- }
-
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/ConfigOption.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/ConfigOption.java
deleted file mode 100644
index a3e140b2bc..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/ConfigOption.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.config;
-
-import java.util.Locale;
-
-public class ConfigOption<T> {
- private final String key;
- private final T defaultValue;
- private final Class<?> clazz;
-
-
- public ConfigOption(String key, Class<?> clazz) {
- this.key = key;
- this.clazz = clazz;
- this.defaultValue = null;
- }
-
- public ConfigOption(String key, T defaultValue, Class<?> clazz) {
- this.key = key;
- this.defaultValue = defaultValue;
- this.clazz = clazz;
- }
-
- public String key() {
- return key;
- }
-
- public boolean hasDefaultValue() {
- return defaultValue != null;
- }
-
- Class<?> getClazz() {
- return clazz;
- }
-
- public T defaultValue() {
- return defaultValue;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- } else if (o != null && o.getClass() == ConfigOption.class) {
- ConfigOption<?> that = (ConfigOption<?>) o;
- return this.key.equals(that.key)
- && (this.defaultValue == null
- ? that.defaultValue == null
- : (that.defaultValue != null
- && this.defaultValue.equals(that.defaultValue)));
- } else {
- return false;
- }
- }
-
- @Override
- public int hashCode() {
- return 31 * key.hashCode()
- + (defaultValue != null ? defaultValue.hashCode() : 0);
- }
-
- @Override
- public String toString() {
- return String.format(Locale.ROOT,
- "Key: '%s' , default: %s",
- key, defaultValue);
- }
-
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/DefaultSecondStorageProperties.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/DefaultSecondStorageProperties.java
deleted file mode 100644
index fca1c357ae..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/DefaultSecondStorageProperties.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.config;
-
-import java.util.Locale;
-import java.util.Optional;
-import java.util.Properties;
-
-import io.kyligence.kap.secondstorage.util.ConvertUtils;
-
-public class DefaultSecondStorageProperties implements SecondStorageProperties {
-
- private final Properties properties = new Properties();
-
- public DefaultSecondStorageProperties(Properties param) {
- properties.putAll(param);
- }
-
- @Override
- public <T> T get(ConfigOption<T> option) {
- return getOptional(option).orElseGet(option::defaultValue);
- }
-
- @Override
- public <T> Optional<T> getOptional(ConfigOption<T> option) {
- Optional<Object> rawValue = getRawValue(option.key());
-
- Class<?> clazz = option.getClazz();
-
- try {
- return rawValue.map(v -> ConvertUtils.convertValue(v, clazz));
- } catch (Exception e) {
- throw new IllegalArgumentException(String.format(Locale.ROOT, "Could not parse value '%s' for key '%s'.",
- rawValue.map(Object::toString).orElse(""), option.key()), e);
- }
- }
-
- @Override
- public Properties getProperties() {
- return properties;
- }
-
- private Optional<Object> getRawValue(String key) {
- if (key == null) {
- throw new NullPointerException("Key must not be null.");
- }
- return Optional.ofNullable(this.properties.get(key));
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/Node.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/Node.java
deleted file mode 100644
index d63ab9a237..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/Node.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.config;
-
-import org.apache.kylin.common.util.EncryptUtil;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-// CALL FROM CORE
-public class Node {
- private String name;
- private String ip;
- private int port;
- private String user;
- private String password;
- @JsonProperty("sshPort")
- private int sshPort;
-
- public Node(String name, String ip, int port, String user, String password) {
- this.name = name;
- this.ip = ip;
- this.port = port;
- this.user = user;
- this.password = password;
- }
-
- public Node(String name, String ip, int port, String user, String password, int sshPort) {
- this.name = name;
- this.ip = ip;
- this.port = port;
- this.user = user;
- this.password = password;
- this.sshPort = sshPort;
- }
-
- public Node(Node node) {
- this(node.name, node.ip, node.port, node.user, node.password);
- }
-
- public Node() {
- }
-
- public String getName() {
- return name;
- }
-
- public Node setName(String name) {
- this.name = name;
- return this;
- }
-
- public String getIp() {
- return ip;
- }
-
- public Node setIp(String ip) {
- this.ip = ip;
- return this;
- }
-
- public int getPort() {
- return port;
- }
-
- public Node setPort(int port) {
- this.port = port;
- return this;
- }
-
- public String getUser() {
- return user;
- }
-
- public Node setUser(String user) {
- this.user = user;
- return this;
- }
-
- public String getPassword() {
- return EncryptUtil.isEncrypted(password) ? EncryptUtil.decryptPassInKylin(password) : password;
- }
-
- public Node setPassword(String password) {
- this.password = password;
- return this;
- }
-
- public int getSSHPort() {
- return sshPort;
- }
-
- public Node setSSHPort(int sshPort) {
- this.sshPort = sshPort;
- return this;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageModelSegment.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageModelSegment.java
deleted file mode 100644
index 3289b2ae47..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageModelSegment.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.config;
-
-import java.util.Map;
-
-public class SecondStorageModelSegment {
- private String uuid;
- private String dateFormat;
- private Map<String, SecondStorageSegment> segmentMap;
-
- public SecondStorageModelSegment() {
- }
-
- public SecondStorageModelSegment(String uuid, String dateFormat, Map<String, SecondStorageSegment> segmentMap) {
- this.uuid = uuid;
- this.dateFormat = dateFormat;
- this.segmentMap = segmentMap;
- }
-
- public String getDateFormat() {
- return dateFormat;
- }
-
- public SecondStorageModelSegment setDateFormat(final String dateFormat) {
- this.dateFormat = dateFormat;
- return this;
- }
-
- public String getUuid() {
- return uuid;
- }
-
- public void setUuid(String uuid) {
- this.uuid = uuid;
- }
-
- public Map<String, SecondStorageSegment> getSegmentMap() {
- return segmentMap;
- }
-
- public void setSegmentMap(Map<String, SecondStorageSegment> segmentMap) {
- this.segmentMap = segmentMap;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageProjectModelSegment.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageProjectModelSegment.java
deleted file mode 100644
index 01cee6c4f6..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageProjectModelSegment.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.config;
-
-import java.util.Map;
-
-public class SecondStorageProjectModelSegment {
- private String project;
- private Map<String, SecondStorageModelSegment> modelSegmentMap;
-
- public String getProject() {
- return project;
- }
-
- public Map<String, SecondStorageModelSegment> getModelSegmentMap() {
- return modelSegmentMap;
- }
-
- public void setProject(String project) {
- this.project = project;
- }
-
- public void setModelSegmentMap(Map<String, SecondStorageModelSegment> modelSegmentMap) {
- this.modelSegmentMap = modelSegmentMap;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageProperties.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageProperties.java
deleted file mode 100644
index ca22158fb0..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageProperties.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.config;
-
-import java.util.Optional;
-import java.util.Properties;
-
-public interface SecondStorageProperties {
- <T> T get(ConfigOption<T> option);
-
- <T> Optional<T> getOptional(ConfigOption<T> option);
-
- Properties getProperties();
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageSegment.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageSegment.java
deleted file mode 100644
index d1f146d453..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/config/SecondStorageSegment.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.config;
-
-import org.apache.kylin.metadata.model.SegmentRange;
-
-public class SecondStorageSegment {
- private String segmentId;
- private SegmentRange segmentRange;
-
- public SecondStorageSegment(String segmentId, SegmentRange segmentRange) {
- this.segmentId = segmentId;
- this.segmentRange = segmentRange;
- }
-
- public String getSegmentId() {
- return segmentId;
- }
-
- public SegmentRange getSegmentRange() {
- return segmentRange;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/database/DatabaseOperator.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/database/DatabaseOperator.java
deleted file mode 100644
index 8133040813..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/database/DatabaseOperator.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.database;
-
-import java.io.Closeable;
-import java.util.List;
-
-public interface DatabaseOperator extends Closeable {
- List<String> listDatabases();
-
- List<String> listTables(String database);
-
- void dropTable(String database, String table);
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/database/QueryOperator.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/database/QueryOperator.java
deleted file mode 100644
index c0f04b65b6..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/database/QueryOperator.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.database;
-
-import java.util.Map;
-import java.util.Set;
-
-public interface QueryOperator {
- Map<String, Object> getQueryMetric(String queryId);
-
- void modifyColumnByCardinality(String database, String destTableName, Set<Integer> secondaryIndex);
-
- void modifyColumnByCardinality(String database, String destTableName, String column, String datatype);
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/AlterTable.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/AlterTable.java
deleted file mode 100644
index e3fc156695..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/AlterTable.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.exp.TableIdentifier;
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-import io.kyligence.kap.secondstorage.ddl.visitor.Renderable;
-
-public class AlterTable extends DDL<AlterTable> {
- private final TableIdentifier table;
- private ManipulatePartition manipulatePartition = null;
- private ManipulateIndex manipulateIndex = null;
- private boolean freeze = false;
- private String attachPart = null;
-
- private ModifyColumn modifyColumn = null;
-
- public AlterTable(TableIdentifier table, ManipulatePartition manipulatePartition) {
- this.table = table;
- this.manipulatePartition = manipulatePartition;
- }
-
- public AlterTable(TableIdentifier table, boolean freeze) {
- this.table = table;
- this.freeze = freeze;
- }
-
- public AlterTable(TableIdentifier table, String attachPart) {
- this.table = table;
- this.attachPart = attachPart;
- }
-
- public AlterTable(TableIdentifier table, ManipulateIndex manipulateIndex) {
- this.table = table;
- this.manipulateIndex = manipulateIndex;
- }
-
- public AlterTable(TableIdentifier table, ModifyColumn modifyColumn) {
- this.table = table;
- this.modifyColumn = modifyColumn;
- }
-
- public TableIdentifier getTable() {
- return table;
- }
-
- public ManipulatePartition getManipulatePartition() {
- return manipulatePartition;
- }
-
- public ManipulateIndex getManipulateIndex() {
- return manipulateIndex;
- }
-
- public ModifyColumn getModifyColumn() {
- return modifyColumn;
- }
-
- public boolean isFreeze() {
- return freeze;
- }
-
- public String getAttachPart() {
- return attachPart;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-
- public enum PartitionOperation {
- MOVE {
- @Override
- public String getOperation() {
- return "MOVE";
- }
- },
- DROP {
- @Override
- public String getOperation() {
- return "DROP";
- }
- };
-
- public abstract String getOperation();
- }
-
- public enum IndexOperation {
- ADD, MATERIALIZE, DROP
- }
-
- public static class ManipulatePartition implements Renderable {
- private final String partition;
- private final TableIdentifier destTable;
- private final PartitionOperation partitionOperation;
-
- public ManipulatePartition(String partition, TableIdentifier destTable, PartitionOperation partitionOperation) {
- this.partition = partition;
- this.destTable = destTable;
- this.partitionOperation = partitionOperation;
- }
-
- public ManipulatePartition(String partition, PartitionOperation partitionOperation) {
- this.partition = partition;
- this.destTable = null;
- this.partitionOperation = partitionOperation;
- }
-
- public String getPartition() {
- return partition;
- }
-
- public TableIdentifier getDestTable() {
- return destTable;
- }
-
- public PartitionOperation getPartitionOperation() {
- return partitionOperation;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
- }
-
- public static class ManipulateIndex implements Renderable {
- private final String name;
- private String column;
- private String expr;
- private int granularity;
- private final IndexOperation indexOperation;
-
- public ManipulateIndex(String name, String column, String expr, int granularity) {
- this.name = name;
- this.column = column;
- this.expr = expr;
- this.granularity = granularity;
- this.indexOperation = IndexOperation.ADD;
- }
-
- public ManipulateIndex(String name, IndexOperation indexOperation) {
- this.name = name;
- this.indexOperation = indexOperation;
- }
-
- public IndexOperation getIndexOperation() {
- return indexOperation;
- }
-
- public int getGranularity() {
- return granularity;
- }
-
- public String getExpr() {
- return expr;
- }
-
- public String getColumn() {
- return this.column;
- }
-
- public String getName() {
- return name;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
- }
-
- public static class ModifyColumn implements Renderable {
- private final String column;
-
- private final String datatype;
-
- public ModifyColumn(String column, String datatype) {
- this.column = column;
- this.datatype = datatype;
- }
-
- public String getColumn() {
- return column;
- }
-
- public String getDatatype() {
- return datatype;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/CreateDatabase.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/CreateDatabase.java
deleted file mode 100644
index de7d6b5cb9..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/CreateDatabase.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class CreateDatabase extends DDL<CreateDatabase> {
-
- private final boolean ifNotExists;
- private final String database;
-
- private CreateDatabase(String database, boolean ifNotExists) {
- this.ifNotExists = ifNotExists;
- this.database = database;
- }
-
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-
- public boolean isIfNotExists() {
- return ifNotExists;
- }
-
- public String getDatabase() {
- return database;
- }
-
- public static CreateDatabase createDatabase(String database) {
- return new CreateDatabase(database, true);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/CreateTable.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/CreateTable.java
deleted file mode 100644
index 11f6d7d41b..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/CreateTable.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import io.kyligence.kap.secondstorage.ddl.exp.ColumnWithType;
-import io.kyligence.kap.secondstorage.ddl.exp.TableIdentifier;
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class CreateTable<T extends CreateTable<T>> extends DDL<T> {
-
- private static class Default extends CreateTable<Default> {
- public Default(TableIdentifier table, boolean ifNotExists) {
- super(table, ifNotExists);
- }
- }
-
- private final boolean ifNotExists;
- private final TableIdentifier table;
- private final List<ColumnWithType> columns;
-
- public CreateTable(TableIdentifier table, boolean ifNotExists) {
- this.ifNotExists = ifNotExists;
- this.table = table;
- this.columns = new ArrayList<>();
- }
-
- public boolean isIfNotExists() {
- return ifNotExists;
- }
-
- public TableIdentifier table() {
- return table;
- }
-
- public List<ColumnWithType> getColumns() {
- return columns;
- }
-
- @SuppressWarnings("unchecked")
- public T columns(Collection<? extends ColumnWithType> fields) {
- columns.addAll(fields);
- return (T) this;
- }
-
- public final T columns(ColumnWithType... fields) {
- return columns(Arrays.asList(fields));
- }
-
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-
- public static CreateTable<Default> create(String database, String table) {
- return new Default(TableIdentifier.table(database, table), true);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DDL.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DDL.java
deleted file mode 100644
index 47a7fdc8c7..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DDL.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import org.apache.commons.lang3.StringUtils;
-
-import io.kyligence.kap.secondstorage.ddl.visitor.DefaultSQLRender;
-import io.kyligence.kap.secondstorage.ddl.visitor.Renderable;
-
-public abstract class DDL<T extends DDL<T>> implements Renderable {
-
- final DefaultSQLRender queryBuilder = new DefaultSQLRender();
-
- public String toSql() {
- return toSql(queryBuilder);
- }
-
- public String toSql(final DefaultSQLRender render) {
- render.reset();
- accept(render);
- return render.getResult().toString();
- }
-
- protected static String tableIdentifier(String database, String table) {
- if (StringUtils.isNotEmpty(database)) {
- return database + '.' + table;
- } else {
- return table;
- }
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DDLOperator.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DDLOperator.java
deleted file mode 100644
index 1730ec264a..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DDLOperator.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.ddl;
-
-import org.apache.kylin.metadata.datatype.DataType;
-
-public interface DDLOperator {
- String getSecondaryIndexType(DataType dataType);
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/Desc.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/Desc.java
deleted file mode 100644
index c55badd30d..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/Desc.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.exp.TableIdentifier;
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class Desc extends DDL<AlterTable> {
- private final TableIdentifier table;
-
- public Desc(TableIdentifier table) {
- this.table = table;
- }
-
- public TableIdentifier getTable() {
- return table;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DropDatabase.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DropDatabase.java
deleted file mode 100644
index 0d3c90ed5f..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DropDatabase.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class DropDatabase extends DDL<DropDatabase> {
-
- private final boolean ifExists;
- private final String db;
-
-
- private DropDatabase(String db, boolean ifExists) {
- this.ifExists = ifExists;
- this.db = db;
- }
-
- public static DropDatabase dropDatabase(String database) {
- return new DropDatabase(database, true);
- }
-
- public boolean isIfExists() {
- return ifExists;
- }
-
- public String db() {
- return db;
- }
-
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DropTable.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DropTable.java
deleted file mode 100644
index ffe6c9bc84..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/DropTable.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.exp.TableIdentifier;
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class DropTable extends DDL<DropTable> {
-
- private final boolean ifExists;
- private final TableIdentifier table;
-
-
- private DropTable(TableIdentifier table, boolean ifExists) {
- this.ifExists = ifExists;
- this.table = table;
- }
-
- public boolean isIfExists() {
- return ifExists;
- }
-
- public TableIdentifier table() {
- return table;
- }
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-
- public static DropTable dropTable(String database, String table) {
- return new DropTable(TableIdentifier.table(database, table), true);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ExistsDatabase.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ExistsDatabase.java
deleted file mode 100644
index 01561f5094..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ExistsDatabase.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class ExistsDatabase extends DDL<ExistsDatabase> {
- private final String database;
-
- public ExistsDatabase(String database) {
- this.database = database;
- }
-
- public String getDatabase() {
- return database;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ExistsTable.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ExistsTable.java
deleted file mode 100644
index 2fdf06204e..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ExistsTable.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.exp.TableIdentifier;
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class ExistsTable extends DDL<ExistsTable> {
- private final TableIdentifier tableIdentifier;
-
- public TableIdentifier getTableIdentifier() {
- return tableIdentifier;
- }
-
- public ExistsTable(TableIdentifier tableIdentifier) {
- this.tableIdentifier = tableIdentifier;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/InsertInto.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/InsertInto.java
deleted file mode 100644
index e213fc3533..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/InsertInto.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import java.sql.SQLException;
-
-import org.apache.commons.collections.map.ListOrderedMap;
-
-import io.kyligence.kap.secondstorage.ddl.exp.TableIdentifier;
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class InsertInto extends DDL<InsertInto> {
-
- private final TableIdentifier table;
- private Select select;
- private final ListOrderedMap columnsValues = new ListOrderedMap();
-
- public InsertInto(TableIdentifier table) {
- this.table = table;
- this.select = null;
- }
-
- public TableIdentifier table() {
- return table;
- }
-
- public InsertInto from(String database, String table) {
- select = new Select(TableIdentifier.table(database, table));
- return this;
- }
-
- public InsertInto set(final String column, final Object value) {
- // if (columnsValues.containsKey(column))
- // throw new QueryGrammarException("Column '" + column
- // + "' has already been set.");
- columnsValues.put(column, value);
- return this;
- }
-
- public InsertInto set(final String column, final Object value, final Object defaultValueIfNull) {
- if (value == null)
- return set(column, defaultValueIfNull);
- else
- return set(column, value);
- }
-
- public Select from() {
- return select;
- }
-
- public ListOrderedMap getColumnsValues() {
- return columnsValues;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-
- // throw SQLException for test
- public static InsertInto insertInto(String database, String table) throws SQLException {
- return new InsertInto(TableIdentifier.table(database, table));
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/RenameTable.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/RenameTable.java
deleted file mode 100644
index 5912e45d7f..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/RenameTable.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.exp.TableIdentifier;
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class RenameTable extends DDL<RenameTable> {
-
- private final TableIdentifier source;
- private TableIdentifier to;
-
- private RenameTable(TableIdentifier source) {
- this.source = source;
- }
-
- public TableIdentifier source() {
- return source;
- }
-
- public RenameTable to(String database, String table) {
- this.to = TableIdentifier.table(database, table);
- return this;
- }
-
- public TableIdentifier to() {
- return to;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-
- public static RenameTable renameSource(String database, String table) {
- return new RenameTable(TableIdentifier.table(database, table));
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/Select.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/Select.java
deleted file mode 100644
index d68506abcc..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/Select.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import io.kyligence.kap.secondstorage.ddl.exp.ColumnWithAlias;
-import io.kyligence.kap.secondstorage.ddl.exp.GroupBy;
-import io.kyligence.kap.secondstorage.ddl.exp.TableIdentifier;
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class Select extends DDL<Select> {
- private final TableIdentifier fromTable;
-
- private final List<ColumnWithAlias> columns = new ArrayList<>();
- private String condition;
- private GroupBy groupby;
-
- public Select(TableIdentifier table) {
- fromTable = table;
- }
-
- public Select column(ColumnWithAlias column) {
- columns.add(column);
- return this;
- }
-
- public Select where(String condition) {
- this.condition = condition;
- return this;
- }
-
- public Select groupby(GroupBy groupby) {
- this.groupby = groupby;
- return this;
- }
-
- public void columns(List<ColumnWithAlias> columns) {
- this.columns.addAll(columns);
- }
-
- public List<ColumnWithAlias> columns() {
- return columns;
- }
-
- public TableIdentifier from() {
- return fromTable;
- }
-
- public String where() {
- return condition;
- }
-
- public GroupBy groupby() {
- return groupby;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowCreateDatabase.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowCreateDatabase.java
deleted file mode 100644
index 71586ece20..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowCreateDatabase.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class ShowCreateDatabase extends DDL<ShowCreateDatabase> {
- private final String database;
-
- public ShowCreateDatabase(String database) {
- this.database = database;
- }
-
- public String getDatabase() {
- return database;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowCreateTable.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowCreateTable.java
deleted file mode 100644
index d2711426e2..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowCreateTable.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.exp.TableIdentifier;
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class ShowCreateTable extends DDL<ShowCreateTable> {
- private final TableIdentifier tableIdentifier;
-
- public TableIdentifier getTableIdentifier() {
- return tableIdentifier;
- }
-
- public ShowCreateTable(TableIdentifier tableIdentifier) {
- this.tableIdentifier = tableIdentifier;
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowDatabases.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowDatabases.java
deleted file mode 100644
index d8188b8763..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowDatabases.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class ShowDatabases extends DDL<ShowDatabases> {
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowTables.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowTables.java
deleted file mode 100644
index 61f9d697f2..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/ShowTables.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl;
-
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-
-public class ShowTables extends DDL<ShowTables> {
- private final String database;
-
- private ShowTables(String database) {
- this.database = database;
- }
-
- public String getDatabase() {
- return database;
- }
-
- public static ShowTables createShowTables(String database) {
- return new ShowTables(database);
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/SkippingIndexChooser.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/SkippingIndexChooser.java
deleted file mode 100644
index 8e0d164e57..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/SkippingIndexChooser.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.ddl;
-
-import static org.apache.kylin.common.exception.ServerErrorCode.SECOND_STORAGE_NOT_SUPPORT_TYPE;
-
-import java.util.Set;
-
-import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.msg.MsgPicker;
-import org.apache.kylin.guava30.shaded.common.collect.ImmutableSet;
-import org.apache.kylin.metadata.datatype.DataType;
-
-import io.kyligence.kap.secondstorage.enums.SkippingIndexType;
-
-public class SkippingIndexChooser {
- private static final Set<String> MINMAX_TYPE = ImmutableSet.of(DataType.TINY_INT, DataType.SMALL_INT,
- DataType.INTEGER, DataType.INT, DataType.BIGINT, DataType.TIMESTAMP, DataType.DATE);
-
- private static final Set<String> BLOOM_FILTER_TYPE = ImmutableSet.of(DataType.VARCHAR, DataType.CHAR,
- DataType.STRING);
-
- private static final Set<String> SET_TYPE = ImmutableSet.of(DataType.BOOLEAN);
-
- private SkippingIndexChooser() {
- // can't new
- }
-
- public static SkippingIndexType getSkippingIndexType(DataType dt) {
- if (isMinMax(dt)) {
- return SkippingIndexType.MINMAX;
- } else if (isSet(dt)) {
- return SkippingIndexType.SET;
- } else if (isBloomFilter(dt)) {
- return SkippingIndexType.BLOOM_FILTER;
- } else {
- throw new KylinException(SECOND_STORAGE_NOT_SUPPORT_TYPE,
- MsgPicker.getMsg().getSecondStorageNotSupportType(dt.getName()));
- }
- }
-
- private static boolean isMinMax(DataType dataType) {
- return MINMAX_TYPE.contains(dataType.getName());
- }
-
- private static boolean isSet(DataType dataType) {
- return SET_TYPE.contains(dataType.getName());
- }
-
- private static boolean isBloomFilter(DataType dataType) {
- return BLOOM_FILTER_TYPE.contains(dataType.getName());
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/ColumnWithAlias.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/ColumnWithAlias.java
deleted file mode 100644
index c007f15e5b..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/ColumnWithAlias.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.ddl.exp;
-
-/**
- * @author neng.liu
- */
-public class ColumnWithAlias {
- private final boolean distinct;
- private final String alias;
- private final String name;
- private final String expr;
-
- private ColumnWithAlias(boolean distinct, String alias, String name, String expr) {
- this.distinct = distinct;
- this.alias = alias;
- this.name = name;
- this.expr = expr;
- }
-
- public static ColumnWithAliasBuilder builder() {
- return new ColumnWithAliasBuilder();
- }
-
- public static class ColumnWithAliasBuilder {
-
- private boolean distinct;
- private String alias;
- private String name;
- private String expr;
-
- public ColumnWithAliasBuilder distinct(boolean distinct) {
- this.distinct = distinct;
- return this;
- }
-
- public ColumnWithAliasBuilder alias(String alias) {
- this.alias = alias;
- return this;
- }
-
- public ColumnWithAliasBuilder name(String name) {
- this.name = name;
- return this;
- }
-
- public ColumnWithAliasBuilder expr(String expr) {
- this.expr = expr;
- return this;
- }
-
- public ColumnWithAlias build() {
- return new ColumnWithAlias(distinct, alias, name, expr);
- }
- }
-
- public boolean isDistinct() {
- return distinct;
- }
-
- public String getAlias() {
- return alias;
- }
-
- public String getName() {
- return name;
- }
-
- public String getExpr() {
- return expr;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/ColumnWithType.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/ColumnWithType.java
deleted file mode 100644
index 4d08e3d864..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/ColumnWithType.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl.exp;
-
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-import io.kyligence.kap.secondstorage.ddl.visitor.Renderable;
-
-public class ColumnWithType implements Renderable {
-
- private final String name;
- private final String type;
- private boolean quote;
- private final boolean nullable;
-
- public ColumnWithType(String name, String type) {
- this(name, type, false, false);
- }
-
- public ColumnWithType(String name, String type, boolean nullable) {
- this(name, type, nullable, false);
- }
-
- public ColumnWithType(String name, String type, boolean nullable, boolean quote) {
- this.name = name;
- this.type = type;
- this.quote = quote;
- this.nullable = nullable;
- }
-
- public String name() {
- return name;
- }
-
- public String type() {
- return type;
- }
-
- public boolean quote() {
- return quote;
- }
-
- public boolean nullable() {
- return nullable;
- }
-
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/GroupBy.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/GroupBy.java
deleted file mode 100644
index d9ba6409ce..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/GroupBy.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.ddl.exp;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class GroupBy {
- private List<ColumnWithAlias> columns = new ArrayList<>();
-
- public GroupBy column(ColumnWithAlias column) {
- this.columns.add(column);
- return this;
- }
-
- public List<ColumnWithAlias> columns() {
- return columns;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/TableIdentifier.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/TableIdentifier.java
deleted file mode 100644
index bb90ed39e1..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/exp/TableIdentifier.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl.exp;
-
-import org.apache.commons.lang3.StringUtils;
-
-import io.kyligence.kap.secondstorage.ddl.visitor.RenderVisitor;
-import io.kyligence.kap.secondstorage.ddl.visitor.Renderable;
-
-public class TableIdentifier implements Renderable {
- public static final char TABLE_ESCAPE = '`';
- private final String database;
- private final String table;
-
- private TableIdentifier(String database, String table) {
- this.database = database;
- this.table = table;
- }
-
- public String table() {
- if (StringUtils.isNotEmpty(database)) {
- return TABLE_ESCAPE + database + TABLE_ESCAPE + '.' + TABLE_ESCAPE + table + TABLE_ESCAPE;
- } else {
- return TABLE_ESCAPE + table + TABLE_ESCAPE;
- }
- }
-
- @Override
- public void accept(RenderVisitor visitor) {
- visitor.visit(this);
- }
-
- public static TableIdentifier table(String database, String table) {
- return new TableIdentifier(database, table);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/BaseRender.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/BaseRender.java
deleted file mode 100644
index e5628a1fa3..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/BaseRender.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl.visitor;
-
-public interface BaseRender extends RenderVisitor{
- default void acceptOrVisitValue(final Object param) {
- if (param instanceof Renderable)
- ((Renderable) param).accept(this);
- else
- visitValue(param);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/DefaultSQLRender.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/DefaultSQLRender.java
deleted file mode 100644
index 9798db7e80..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/DefaultSQLRender.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl.visitor;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections.map.ListOrderedMap;
-import org.apache.commons.lang3.StringUtils;
-
-import io.kyligence.kap.secondstorage.ddl.AlterTable;
-import io.kyligence.kap.secondstorage.ddl.CreateDatabase;
-import io.kyligence.kap.secondstorage.ddl.CreateTable;
-import io.kyligence.kap.secondstorage.ddl.Desc;
-import io.kyligence.kap.secondstorage.ddl.DropDatabase;
-import io.kyligence.kap.secondstorage.ddl.DropTable;
-import io.kyligence.kap.secondstorage.ddl.ExistsDatabase;
-import io.kyligence.kap.secondstorage.ddl.ExistsTable;
-import io.kyligence.kap.secondstorage.ddl.InsertInto;
-import io.kyligence.kap.secondstorage.ddl.RenameTable;
-import io.kyligence.kap.secondstorage.ddl.Select;
-import io.kyligence.kap.secondstorage.ddl.ShowCreateDatabase;
-import io.kyligence.kap.secondstorage.ddl.ShowCreateTable;
-import io.kyligence.kap.secondstorage.ddl.ShowDatabases;
-import io.kyligence.kap.secondstorage.ddl.ShowTables;
-import io.kyligence.kap.secondstorage.ddl.exp.ColumnWithAlias;
-import io.kyligence.kap.secondstorage.ddl.exp.ColumnWithType;
-import io.kyligence.kap.secondstorage.ddl.exp.GroupBy;
-import io.kyligence.kap.secondstorage.ddl.exp.TableIdentifier;
-
-public class DefaultSQLRender implements BaseRender {
-
- protected static final String OPEN_BRACKET = "(";
- protected static final String CLOSE_BRACKET = ")";
-
- @Override
- public void visit(Select select) {
- result.append(KeyWord.SELECT);
- if (select.columns().isEmpty()) {
- result.append(" * ");
- } else {
- result.append(' ');
- List<String> columns = new ArrayList<>(select.columns().size());
- for (ColumnWithAlias column : select.columns()) {
- columns.add(buildColumnWithAlias(column));
- }
- result.append(String.join(",", columns));
- result.append(' ');
- }
-
- result.append(KeyWord.FROM);
- acceptOrVisitValue(select.from());
- if (select.where() != null) {
- result.append(' ').append(KeyWord.WHERE).append(' ').append(select.where());
- }
- if (select.groupby() != null) {
- result.append(' ');
- this.visit(select.groupby());
- }
- }
-
- private String buildColumnWithAlias(ColumnWithAlias column) {
- StringBuilder columnString = new StringBuilder();
- if (column.isDistinct()) {
- columnString.append(KeyWord.DISTINCT);
- }
- if (column.getExpr() != null) {
- columnString.append(' ').append(column.getExpr());
- } else {
- columnString.append(" `").append(column.getName()).append("`");
- }
- if (column.getAlias() != null) {
- columnString.append(" AS ").append(column.getAlias());
- }
- return columnString.toString();
- }
-
- @Override
- public void visit(GroupBy groupBy) {
- result.append(KeyWord.GROUP_BY);
- result.append(' ');
- List<String> columns = groupBy.columns().stream().map(col -> "`" + col.getName() + "`")
- .collect(Collectors.toList());
- result.append(String.join(",", columns));
- }
-
- protected final StringBuilder result = new StringBuilder();
-
- // dialect
- protected String quoteColumn(String col) {
- return "`" + col + "`";
- }
-
- private String nullableColumn(String type) {
- return KeyWord.NULLABLE + OPEN_BRACKET + type + CLOSE_BRACKET;
- }
-
- public StringBuilder getResult() {
- return result;
- }
-
- public void reset() {
- result.setLength(0);
- }
-
- @Override
- public void visit(ColumnWithType column) {
- if (column.quote()) {
- result.append(quoteColumn(column.name()));
- } else {
- result.append(column.name());
- }
- String type = column.type();
- if (column.nullable()) {
- type = nullableColumn(type);
- }
- result.append(' ').append(type);
- }
-
- @Override
- public void visit(TableIdentifier tableIdentifier) {
- result.append(' ').append(tableIdentifier.table());
- }
-
- @Override
- public void visit(RenameTable renameTable) {
- result.append(KeyWord.RENAME).append(' ').append(KeyWord.TABLE);
- acceptOrVisitValue(renameTable.source());
- result.append(' ').append(KeyWord.TO);
- acceptOrVisitValue(renameTable.to());
- }
-
- protected void createTablePrefix(CreateTable<?> query) {
- result.append(KeyWord.CREATE).append(' ').append(KeyWord.TABLE);
- if (query.isIfNotExists()) {
- result.append(' ').append(KeyWord.IF_NOT_EXISTS);
- }
- acceptOrVisitValue(query.table());
- }
-
- @Override
- public void visit(CreateTable<?> query) {
- createTablePrefix(query);
-
- result.append('(');
- boolean firstColumn = true;
- for (final ColumnWithType column : query.getColumns()) {
- if (firstColumn)
- firstColumn = false;
- else
- result.append(",");
- acceptOrVisitValue(column);
- }
- result.append(')');
- }
-
- @Override
- public void visit(CreateDatabase createDatabase) {
- result.append(KeyWord.CREATE).append(' ').append(KeyWord.DATABASE);
- if (createDatabase.isIfNotExists()) {
- result.append(' ').append(KeyWord.IF_NOT_EXISTS);
- }
- result.append(' ').append(createDatabase.getDatabase());
- }
-
- @Override
- public void visit(DropTable dropTable) {
- result.append(KeyWord.DROP).append(' ').append(KeyWord.TABLE);
- if (dropTable.isIfExists()) {
- result.append(' ').append(KeyWord.IF_EXISTS);
- }
- acceptOrVisitValue(dropTable.table());
- }
-
- @Override
- public void visit(DropDatabase dropDatabase) {
- result.append(KeyWord.DROP).append(' ').append(KeyWord.DATABASE);
- if (dropDatabase.isIfExists()) {
- result.append(' ').append(KeyWord.IF_EXISTS);
- }
- result.append(' ').append(dropDatabase.db());
- }
-
- @Override
- public void visit(InsertInto insert) {
- result.append(KeyWord.INSERT);
- acceptOrVisitValue(insert.table());
- result.append(' ');
- if (insert.from() != null) {
- acceptOrVisitValue(insert.from());
- } else {
- result.append(OPEN_BRACKET);
- final ListOrderedMap columnValues = insert.getColumnsValues();
- result.append(StringUtils.join(columnValues.keyList(), ", "));
- result.append(CLOSE_BRACKET).append(" ");
- result.append(KeyWord.VALUES).append(" ").append(OPEN_BRACKET);
- boolean firstClause = true;
- for (final Object column : columnValues.keyList()) {
- if (firstClause) {
- firstClause = false;
- } else {
- result.append(", ");
- }
- acceptOrVisitValue(columnValues.get(column));
- }
- result.append(CLOSE_BRACKET);
- }
- }
-
- @Override
- public void visit(ShowCreateDatabase showCreateDatabase) {
- result.append(KeyWord.SHOW_CREATE_DATABASE).append(' ').append(showCreateDatabase.getDatabase());
- }
-
- @Override
- public void visit(ShowCreateTable showCreateTable) {
- result.append(KeyWord.SHOW_CREATE_TABLE).append(' ').append(showCreateTable.getTableIdentifier().table());
- }
-
- @Override
- public void visit(ExistsDatabase existsDatabase) {
- result.append(KeyWord.EXISTS).append(" ").append(KeyWord.DATABASE).append(" ")
- .append(existsDatabase.getDatabase());
- }
-
- @Override
- public void visit(ExistsTable existsTable) {
- result.append(KeyWord.EXISTS).append(" ").append(KeyWord.TABLE).append(" ")
- .append(existsTable.getTableIdentifier().table());
- }
-
- @Override
- public void visit(ShowDatabases showDatabases) {
- result.append("SHOW DATABASES");
- }
-
- @Override
- public void visit(ShowTables showTables) {
- result.append("SHOW TABLES").append(" FROM ").append(showTables.getDatabase());
- }
-
- @Override
- public void visit(AlterTable alterTable) {
- throw new UnsupportedOperationException("Alter table doesn't support by default render");
- }
-
- protected static class KeyWord {
- private KeyWord() {
- }
-
- public static final String CREATE = "CREATE";
- public static final String DATABASE = "DATABASE";
- public static final String SHOW_CREATE_DATABASE = "SHOW CREATE DATABASE";
- public static final String TABLE = "TABLE";
- public static final String DISTINCT = "DISTINCT";
- public static final String ALTER = "ALTER";
- public static final String SHOW_CREATE_TABLE = "SHOW CREATE TABLE";
- public static final String EXISTS = "EXISTS";
- public static final String WHERE = "WHERE";
- public static final String GROUP_BY = "GROUP BY";
-
- public static final String DROP = "DROP";
- public static final String IF_EXISTS = "if exists";
- public static final String IF_NOT_EXISTS = "if not exists";
-
- public static final String INSERT = "INSERT INTO";
- public static final String SELECT = "SELECT";
- public static final String FROM = "FROM";
-
- public static final String AS = "AS";
- public static final String ORDER_BY = "ORDER BY";
-
- public static final String RENAME = "RENAME";
- public static final String TO = "TO";
-
- private static final String VALUES = "VALUES";
- private static final String NULLABLE = "Nullable";
-
- private static final String DESC = "DESC";
- }
-
- @Override
- public void visit(AlterTable.ManipulatePartition movePartition) {
- throw new UnsupportedOperationException("Move Partition doesn't support by default render");
- }
-
- @Override
- public void visit(AlterTable.ManipulateIndex manipulateIndex) {
- throw new UnsupportedOperationException("ALTER INDEX doesn't support by default render");
- }
-
- @Override
- public void visit(AlterTable.ModifyColumn modifyColumn) {
- throw new UnsupportedOperationException("Modify Column doesn't support by default render");
- }
-
- @Override
- public void visit(Desc desc) {
- result.append(KeyWord.DESC).append(' ').append(desc.getTable().table());
- }
-
- @Override
- public void visitValue(Object pram) {
- if (pram instanceof String) {
- result.append("'").append(pram).append("'");
- } else {
- result.append(pram);
- }
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/RenderVisitor.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/RenderVisitor.java
deleted file mode 100644
index 14b4e79fe9..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/RenderVisitor.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl.visitor;
-
-
-import io.kyligence.kap.secondstorage.ddl.AlterTable;
-import io.kyligence.kap.secondstorage.ddl.CreateDatabase;
-import io.kyligence.kap.secondstorage.ddl.CreateTable;
-import io.kyligence.kap.secondstorage.ddl.Desc;
-import io.kyligence.kap.secondstorage.ddl.DropDatabase;
-import io.kyligence.kap.secondstorage.ddl.DropTable;
-import io.kyligence.kap.secondstorage.ddl.ExistsDatabase;
-import io.kyligence.kap.secondstorage.ddl.ExistsTable;
-import io.kyligence.kap.secondstorage.ddl.InsertInto;
-import io.kyligence.kap.secondstorage.ddl.RenameTable;
-import io.kyligence.kap.secondstorage.ddl.Select;
-import io.kyligence.kap.secondstorage.ddl.ShowCreateDatabase;
-import io.kyligence.kap.secondstorage.ddl.ShowCreateTable;
-import io.kyligence.kap.secondstorage.ddl.ShowDatabases;
-import io.kyligence.kap.secondstorage.ddl.ShowTables;
-import io.kyligence.kap.secondstorage.ddl.exp.ColumnWithType;
-import io.kyligence.kap.secondstorage.ddl.exp.GroupBy;
-import io.kyligence.kap.secondstorage.ddl.exp.TableIdentifier;
-
-public interface RenderVisitor {
- void visit(ColumnWithType column);
- void visit(TableIdentifier tableIdentifier);
- void visit(RenameTable renameTable);
- void visit(CreateTable<?> createTable);
- void visit(CreateDatabase createDatabase);
- void visit(ShowCreateDatabase showCreateDatabase);
- void visit(DropTable dropTable);
- void visit(DropDatabase dropDatabase);
- void visit(InsertInto insert);
- void visit(Select insert);
- void visit(GroupBy groupBy);
- void visit(AlterTable alterTable);
- void visit(ShowCreateTable showCreateTable);
- void visit(ExistsDatabase existsDatabase);
- void visit(ExistsTable existsTable);
- void visit(ShowDatabases showDatabases);
- void visit(ShowTables showTables);
-
- void visitValue(Object pram);
-
- void visit(AlterTable.ManipulatePartition movePartition);
-
- void visit(AlterTable.ManipulateIndex manipulateIndex);
-
- void visit(AlterTable.ModifyColumn modifyColumn);
-
- void visit(Desc desc);
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/Renderable.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/Renderable.java
deleted file mode 100644
index 5582abc26d..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/ddl/visitor/Renderable.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.ddl.visitor;
-
-public interface Renderable {
- void accept(final RenderVisitor visitor);
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/enums/LockOperateTypeEnum.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/enums/LockOperateTypeEnum.java
deleted file mode 100644
index 57116b36ae..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/enums/LockOperateTypeEnum.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.enums;
-
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY;
-
-import org.apache.kylin.common.exception.KylinException;
-
-public enum LockOperateTypeEnum {
- LOCK, UNLOCK;
-
- public static LockOperateTypeEnum parse(String value) {
- if (value == null) {
- return null;
- }
- for (LockOperateTypeEnum typeEnum : LockOperateTypeEnum.values()) {
- if (typeEnum.name().equals(value)) {
- return typeEnum;
- }
- }
- return null;
- }
-
- public static void check(String lockOperateType) {
- LockOperateTypeEnum typeEnum = LockOperateTypeEnum.parse(lockOperateType);
- if (typeEnum == null) {
- throw new KylinException(REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY, "lockOperateType");
- }
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/enums/LockTypeEnum.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/enums/LockTypeEnum.java
deleted file mode 100644
index c591646d42..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/enums/LockTypeEnum.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.enums;
-
-import static org.apache.kylin.common.exception.ServerErrorCode.SECOND_STORAGE_PROJECT_LOCKING;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.PARAMETER_INVALID_SUPPORT_LIST;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.msg.MsgPicker;
-
-// CALL FROM CORE
-public enum LockTypeEnum {
- QUERY, LOAD, ALL;
-
- public static List<String> subtract(List<String> existLockTypes, List<String> newLockTypes) {
- if (CollectionUtils.isEmpty(existLockTypes)) {
- return new ArrayList<>();
- } else if (CollectionUtils.isEmpty(newLockTypes)) {
- return new ArrayList<>(existLockTypes);
- } else if (newLockTypes.contains(LockTypeEnum.ALL.name())) {
- return new ArrayList<>();
- } else {
- List<String> resultList = new ArrayList<>();
- existLockTypes.forEach(x -> {
- if (!newLockTypes.contains(x)) {
- resultList.add(x);
- }
- });
- return resultList;
- }
- }
-
- public static List<String> merge(List<String> existLockTypes, List<String> newLockTypes) {
- if (CollectionUtils.isEmpty(existLockTypes)) {
- return new ArrayList<>(newLockTypes);
- } else if (CollectionUtils.isEmpty(newLockTypes)) {
- return new ArrayList<>(existLockTypes);
- } else if (existLockTypes.contains(LockTypeEnum.ALL.name()) || newLockTypes.contains(LockTypeEnum.ALL.name())) {
- return Collections.singletonList(LockTypeEnum.ALL.name());
- } else {
- List<String> resultList = new ArrayList<>(existLockTypes);
- newLockTypes.forEach(x -> {
- if (!resultList.contains(x)) {
- resultList.add(x);
- }
- });
- return resultList;
- }
- }
-
- public static LockTypeEnum parse(String value) {
- if (value == null) {
- return null;
- }
- for (LockTypeEnum lockTypeEnum : LockTypeEnum.values()) {
- if (lockTypeEnum.name().equals(value)) {
- return lockTypeEnum;
- }
- }
- return null;
- }
-
- public static boolean locked(String requestLock, List<String> existLocks) {
- if (requestLock == null)
- return false;
- return locked(Collections.singletonList(requestLock), existLocks);
- }
-
- public static boolean locked(List<String> requestLocks, List<String> existLocks) {
- if (CollectionUtils.isEmpty(requestLocks) || CollectionUtils.isEmpty(existLocks)) {
- return false;
- }
-
- Set<String> requestLockSet = new HashSet<>(requestLocks);
-
- if (requestLockSet.contains(LockTypeEnum.ALL.name()) && !existLocks.isEmpty()) {
- return true;
- }
-
- Set<String> existLockSet = new HashSet<>(existLocks);
-
- return existLockSet.contains(LockTypeEnum.ALL.name())
- || CollectionUtils.intersection(requestLockSet, existLockSet).size() > 0;
- }
-
- public static void checkLock(String requestLock, List<String> existLocks) {
- if (requestLock == null)
- return;
- if (locked(Collections.singletonList(requestLock), existLocks)) {
- throw new KylinException(SECOND_STORAGE_PROJECT_LOCKING,
- String.format(Locale.ROOT, MsgPicker.getMsg().getProjectLocked()));
- }
- }
-
- public static void checkLocks(List<String> requestLocks, List<String> existLocks) {
- if (locked(requestLocks, existLocks)) {
- throw new KylinException(SECOND_STORAGE_PROJECT_LOCKING,
- String.format(Locale.ROOT, MsgPicker.getMsg().getProjectLocked()));
- }
- }
-
- public static void check(List<String> lockTypes) {
- if (lockTypes == null || CollectionUtils.isEmpty(lockTypes)) {
- throw new KylinException(REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY, "lockType");
- }
- lockTypes.forEach(x -> {
- LockTypeEnum typeEnum = LockTypeEnum.parse(x);
- if (typeEnum == null) {
- throw new KylinException(PARAMETER_INVALID_SUPPORT_LIST, "lockType", "QUERY, LOAD, ALL");
- }
- });
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/enums/SkippingIndexType.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/enums/SkippingIndexType.java
deleted file mode 100644
index 632e1c70b7..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/enums/SkippingIndexType.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.enums;
-
-import java.util.Locale;
-
-import org.apache.kylin.common.KylinConfig;
-
-public enum SkippingIndexType {
- BLOOM_FILTER {
- private static final String FUNCTION_BLOOM_FILTER = "bloom_filter(%s)";
-
- @Override
- public String toSql(KylinConfig modelConfig) {
- return String.format(Locale.ROOT, FUNCTION_BLOOM_FILTER,
- modelConfig.getSecondStorageSkippingIndexBloomFilter());
- }
- },
-
- SET {
- private static final String FUNCTION_SET = "set(%d)";
-
- @Override
- public String toSql(KylinConfig modelConfig) {
- return String.format(Locale.ROOT, FUNCTION_SET, modelConfig.getSecondStorageSkippingIndexSet());
- }
- },
-
- MINMAX {
- private static final String FUNCTION_MINMAX = "minmax()";
-
- @Override
- public String toSql(KylinConfig modelConfig) {
- return FUNCTION_MINMAX;
- }
- };
-
- public abstract String toSql(KylinConfig modelConfig);
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageDatabaseOperatorFactory.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageDatabaseOperatorFactory.java
deleted file mode 100644
index 5fe8670d8a..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageDatabaseOperatorFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.factory;
-
-import io.kyligence.kap.secondstorage.database.DatabaseOperator;
-
-public interface SecondStorageDatabaseOperatorFactory extends SecondStorageFactory {
- DatabaseOperator createDatabaseOperator(String jdbcUrl);
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageFactory.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageFactory.java
deleted file mode 100644
index 89a7080f35..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageFactory.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.factory;
-
-public interface SecondStorageFactory {
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageFactoryUtils.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageFactoryUtils.java
deleted file mode 100644
index 1dbb48a6fc..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageFactoryUtils.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.factory;
-
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
-import io.kyligence.kap.secondstorage.config.SecondStorageProperties;
-import io.kyligence.kap.secondstorage.database.DatabaseOperator;
-import io.kyligence.kap.secondstorage.database.QueryOperator;
-import io.kyligence.kap.secondstorage.ddl.DDLOperator;
-import io.kyligence.kap.secondstorage.metadata.MetadataOperator;
-
-public class SecondStorageFactoryUtils {
- private static final Map<Class<? extends SecondStorageFactory>, SecondStorageFactory> FACTORY_MAP //
- = new ConcurrentHashMap<>();
-
- private SecondStorageFactoryUtils() {
- // can't new
- }
-
- public static void register(Class<? extends SecondStorageFactory> type, SecondStorageFactory factory) {
- Preconditions.checkArgument(type.isAssignableFrom(factory.getClass()), String.format(Locale.ROOT,
- "type %s is not assignable from %s", type.getName(), factory.getClass().getName()));
- FACTORY_MAP.put(type, factory);
- }
-
- public static MetadataOperator createMetadataOperator(SecondStorageProperties properties) {
- SecondStorageMetadataFactory factory = (SecondStorageMetadataFactory) FACTORY_MAP
- .get(SecondStorageMetadataFactory.class);
- return factory.createMetadataOperator(properties);
- }
-
- public static DatabaseOperator createDatabaseOperator(String jdbcUrl) {
- SecondStorageDatabaseOperatorFactory factory = (SecondStorageDatabaseOperatorFactory) FACTORY_MAP
- .get(SecondStorageDatabaseOperatorFactory.class);
- return factory.createDatabaseOperator(jdbcUrl);
- }
-
- public static QueryOperator createQueryMetricOperator(String project) {
- SecondStorageQueryOperatorFactory factory = (SecondStorageQueryOperatorFactory) FACTORY_MAP
- .get(SecondStorageQueryOperatorFactory.class);
- return factory.getQueryOperator(project);
- }
-
- public static DDLOperator createSecondaryDDLOperator() {
- SecondStorageIndexFactory factory = (SecondStorageIndexFactory) FACTORY_MAP
- .get(SecondStorageIndexFactory.class);
- return factory.createDDLOperator();
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageIndexFactory.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageIndexFactory.java
deleted file mode 100644
index afea2d18f1..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageIndexFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.factory;
-
-import io.kyligence.kap.secondstorage.ddl.DDLOperator;
-
-public interface SecondStorageIndexFactory extends SecondStorageFactory {
- DDLOperator createDDLOperator();
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageMetadataFactory.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageMetadataFactory.java
deleted file mode 100644
index 0de0eda604..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageMetadataFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.factory;
-
-import io.kyligence.kap.secondstorage.config.SecondStorageProperties;
-import io.kyligence.kap.secondstorage.metadata.MetadataOperator;
-
-public interface SecondStorageMetadataFactory extends SecondStorageFactory {
- MetadataOperator createMetadataOperator(SecondStorageProperties properties);
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageQueryOperatorFactory.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageQueryOperatorFactory.java
deleted file mode 100644
index 0f4ab5779a..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/factory/SecondStorageQueryOperatorFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.factory;
-
-import io.kyligence.kap.secondstorage.database.QueryOperator;
-
-public interface SecondStorageQueryOperatorFactory extends SecondStorageFactory {
- QueryOperator getQueryOperator(String project);
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/HasLayoutElement.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/HasLayoutElement.java
deleted file mode 100644
index 1ebf1cf9e7..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/HasLayoutElement.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-import org.apache.kylin.metadata.cube.model.LayoutEntity;
-
-import java.util.List;
-import java.util.Optional;
-
-public interface HasLayoutElement<E extends WithLayout> {
-
- // utility
- static <E extends WithLayout> boolean sameLayout(E e, LayoutEntity layoutEntity){
- return sameLayout(e, layoutEntity.getId());
- }
-
- static <E extends WithLayout> boolean sameLayout(E e, long layoutId){
- return e.getLayoutID() == layoutId;
- }
-
- static <E extends WithLayout> boolean sameIndex(E e, LayoutEntity layoutEntity){
- return layoutEntity.getIndexId() == e.getLayoutID() / 10000 * 10000;
- }
-
- List<E> all();
-
- default Optional<E> getEntity(LayoutEntity layoutEntity, boolean sameLayout){
- return all().stream()
- .filter(e -> sameLayout ? sameLayout(e, layoutEntity): sameIndex(e, layoutEntity))
- .findFirst();
- }
-
- default Optional<E> getEntity(LayoutEntity layoutEntity) {
- return getEntity(layoutEntity.getId());
- }
-
- default Optional<E> getEntity(long layout) {
- return all().stream()
- .filter(e -> sameLayout(e, layout))
- .findFirst();
- }
-
- default boolean containIndex(LayoutEntity layoutEntity, boolean throwOnDifferentLayout){
- return getEntity(layoutEntity, false)
- .map(e -> {
- if (!throwOnDifferentLayout || sameLayout(e, layoutEntity))
- return true;
- else
- throw new IllegalStateException("");
- })
- .orElse(false);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/IManager.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/IManager.java
deleted file mode 100644
index 8396dd4367..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/IManager.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.slf4j.Logger;
-
-public interface IManager<T extends RootPersistentEntity> {
- Logger logger();
-
- String name();
-
- String rootPath();
-
- Class<T> entityType();
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/IManagerAware.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/IManagerAware.java
deleted file mode 100644
index 09b60a167d..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/IManagerAware.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-
-public interface IManagerAware <T extends RootPersistentEntity> {
- void setManager(Manager<T> manager);
- void verify();
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java
deleted file mode 100644
index 5e14ee75e7..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/Manager.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Consumer;
-import java.util.function.Supplier;
-
-import javax.annotation.concurrent.NotThreadSafe;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.persistence.transaction.UnitOfWork;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.metadata.cachesync.CachedCrudAssist;
-
-// CALL FROM CORE
-@NotThreadSafe
-public abstract class Manager<T extends RootPersistentEntity> implements IManager<T> {
-
- protected final KylinConfig config;
- protected final String project;
- protected final CachedCrudAssist<T> crud;
-
- protected ResourceStore getStore() {
- return ResourceStore.getKylinMetaStore(this.config);
- }
-
- protected Manager(KylinConfig cfg, final String project) {
-
- if (logger().isInfoEnabled() && !UnitOfWork.isAlreadyInTransaction()) {
- logger().info("Initializing {} with KylinConfig Id: {} for project {}", name(),
- System.identityHashCode(cfg), project);
- }
-
- this.config = cfg;
- this.project = project;
- this.crud = new CachedCrudAssist<T>(getStore(), rootPath(), entityType()) {
- @SuppressWarnings("unchecked")
- @Override
- protected T initEntityAfterReload(T entity, String resourceName) {
- if (entity instanceof IManagerAware) {
- IManagerAware<T> managerAware = (IManagerAware<T>) entity;
- managerAware.setManager(Manager.this);
- }
- return entity;
- }
- };
- this.crud.setCheckCopyOnWrite(true);
- }
-
- public String getResourcePath(String resourceName) {
- return crud.resourcePath(resourceName);
- }
-
- @SuppressWarnings("unchecked")
- protected T save(T entity) {
- if (entity instanceof IManagerAware) {
- IManagerAware<T> managerAware = (IManagerAware<T>) entity;
- managerAware.verify();
- }
- return crud.save(entity);
- }
-
- protected T copy(T entity) {
- return crud.copyBySerialization(entity);
- }
-
- protected T copyForWrite(T entity) {
- return crud.copyForWrite(entity);
- }
-
- // Create
- protected abstract T newRootEntity(String cubeName);
-
- public T makeSureRootEntity(String cubeName) {
- return get(cubeName).orElseGet(() -> createAS(newRootEntity(cubeName)));
- }
-
- public T createAS(T entity) {
- if (entity.getUuid() == null)
- throw new IllegalArgumentException();
- T copy = copyForWrite(entity);
- if (crud.contains(copy.getUuid()))
- throw new IllegalArgumentException("Entity '" + entity.getUuid() + "' already exists");
- // overwrite point
- return save(copy);
- }
-
- // Read
- public List<T> listAll() {
- return Lists.newArrayList(crud.listAll());
- }
-
- public Optional<T> get(String uuid) {
- if (StringUtils.isEmpty(uuid)) {
- return Optional.empty();
- }
- return Optional.ofNullable(crud.get(uuid));
- }
-
- //Update
- protected T internalUpdate(T entity) {
- if (entity.isCachedAndShared())
- throw new IllegalStateException();
-
- if (entity.getUuid() == null)
- throw new IllegalArgumentException();
-
- String name = entity.getUuid();
- if (!crud.contains(name))
- throw new IllegalArgumentException("Entity '" + name + "' does not exist.");
-
- return save(entity);
- }
-
- public T update(String uuid, Consumer<T> updater) {
- return get(uuid).map(this::copyForWrite).map(copied -> {
- updater.accept(copied);
- return internalUpdate(copied);
- }).orElse(null);
- }
-
- protected T upsert(String uuid, Consumer<T> updater, Supplier<T> creator) {
- return get(uuid).map(this::copyForWrite).map(copied -> {
- updater.accept(copied);
- return internalUpdate(copied);
- }).orElseGet(() -> {
- T newEntity = creator.get();
- updater.accept(newEntity);
- return createAS(newEntity);
- });
- }
-
- //Delete
- public void delete(T entity) {
- crud.delete(entity);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/MetadataOperator.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/MetadataOperator.java
deleted file mode 100644
index 23608e84cb..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/MetadataOperator.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-import io.kyligence.kap.secondstorage.response.SizeInNodeResponse;
-import io.kyligence.kap.secondstorage.response.TableSyncResponse;
-
-public interface MetadataOperator {
- TableSyncResponse tableSync();
-
- SizeInNodeResponse sizeInNode();
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/NodeGroup.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/NodeGroup.java
deleted file mode 100644
index d328d43104..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/NodeGroup.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.metadata;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.function.Consumer;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.springframework.util.CollectionUtils;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-
-import io.kyligence.kap.secondstorage.metadata.annotation.DataDefinition;
-
-// CALL FROM CORE
-@DataDefinition
-public class NodeGroup extends RootPersistentEntity implements Serializable,
- IManagerAware<NodeGroup> {
- @JsonProperty("nodeNames")
- private List<String> nodeNames = new ArrayList<>();
- @JsonProperty("lock_types")
- private List<String> lockTypes = new ArrayList<>();
- private transient Manager<NodeGroup> manager;
-
- public static Builder builder() {
- return new Builder();
- }
-
- @Override
- public void setManager(Manager<NodeGroup> manager) {
- this.manager = manager;
- }
-
- @Override
- public void verify() {
- // node name can't duplicate
- if (nodeNames != null) {
- Preconditions.checkArgument(nodeNames.stream().distinct().count() == nodeNames.size());
- }
- if (lockTypes != null) {
- Preconditions.checkArgument(lockTypes.stream().distinct().count() == lockTypes.size());
- }
- }
-
- public List<String> getNodeNames() {
- return CollectionUtils.isEmpty(nodeNames) ? Collections.emptyList() : Collections.unmodifiableList(nodeNames);
- }
-
- public NodeGroup setNodeNames(List<String> nodeNames) {
- this.checkIsNotCachedAndShared();
- this.nodeNames = nodeNames;
- return this;
- }
-
- public List<String> getLockTypes() {
- return CollectionUtils.isEmpty(lockTypes) ? Collections.emptyList() : Collections.unmodifiableList(lockTypes);
- }
-
- public NodeGroup setLockTypes(List<String> lockTypes) {
- this.lockTypes = lockTypes;
- return this;
- }
-
- @Override
- public String getResourcePath() {
- return manager.getResourcePath(resourceName());
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- if (!super.equals(o)) return false;
-
- NodeGroup nodeGroup = (NodeGroup) o;
-
- return nodeNames != null ? nodeNames.equals(nodeGroup.nodeNames) : nodeGroup.nodeNames == null;
- }
-
- @Override
- public int hashCode() {
- int result = super.hashCode();
- result = 31 * result + (nodeNames != null ? nodeNames.hashCode() : 0);
- return result;
- }
-
- public NodeGroup update(Consumer<NodeGroup> updater) {
- Preconditions.checkArgument(manager != null);
- return manager.update(uuid, updater);
- }
-
- public static final class Builder {
- private List<String> nodeNames;
- private List<String> lockTypes;
-
- public Builder setNodeNames(List<String> nodeNames) {
- this.nodeNames = nodeNames;
- return this;
- }
-
- public Builder setLockTypes(List<String> lockTypes) {
- this.lockTypes = lockTypes;
- return this;
- }
-
- public NodeGroup build() {
- NodeGroup nodeGroup = new NodeGroup();
- nodeGroup.nodeNames = nodeNames;
- nodeGroup.lockTypes = lockTypes;
- return nodeGroup;
- }
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/PartitionType.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/PartitionType.java
deleted file mode 100644
index 2e5f5641a0..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/PartitionType.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-public enum PartitionType {
- FULL,
- INCREMENTAL
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/SegmentFileStatus.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/SegmentFileStatus.java
deleted file mode 100644
index c7879bb3d1..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/SegmentFileStatus.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.metadata;
-
-import java.io.Serializable;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-@JsonAutoDetect(
- fieldVisibility = JsonAutoDetect.Visibility.NONE,
- getterVisibility = JsonAutoDetect.Visibility.NONE,
- isGetterVisibility = JsonAutoDetect.Visibility.NONE,
- setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class SegmentFileStatus implements Serializable {
- @JsonProperty("path")
- private String path;
- /**
- * file length in bytes
- */
- @JsonProperty("len")
- private Long len;
-
- public static SegmentFileStatusBuilder builder() {
- return new SegmentFileStatusBuilder();
- }
-
- public String getPath() {
- return path;
- }
-
- public Long getLen() {
- return len;
- }
-
- public static class SegmentFileStatusBuilder {
- private String path;
- private Long len;
-
- public SegmentFileStatusBuilder setPath(String path) {
- this.path = path;
- return this;
- }
-
- public SegmentFileStatusBuilder setLen(Long len) {
- this.len = len;
- return this;
- }
-
- public SegmentFileStatus build() {
- SegmentFileStatus status = new SegmentFileStatus();
- status.len = len;
- status.path = path;
- return status;
- }
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableData.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableData.java
deleted file mode 100644
index b0f8758e9f..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableData.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.kylin.common.QueryContext;
-import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.exception.ServerErrorCode;
-import org.apache.kylin.common.msg.MsgPicker;
-import org.apache.kylin.common.util.RandomUtil;
-import org.apache.kylin.metadata.cube.model.LayoutEntity;
-import org.apache.spark.sql.execution.datasources.jdbc.ShardOptions$;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-
-import io.kyligence.kap.secondstorage.SecondStorageNodeHelper;
-import io.kyligence.kap.secondstorage.SecondStorageQueryRouteUtil;
-import io.kyligence.kap.secondstorage.metadata.annotation.DataDefinition;
-import scala.collection.JavaConverters;
-
-// CALL FROM CORE
-@JsonAutoDetect(
- fieldVisibility = JsonAutoDetect.Visibility.NONE,
- getterVisibility = JsonAutoDetect.Visibility.NONE,
- isGetterVisibility = JsonAutoDetect.Visibility.NONE,
- setterVisibility = JsonAutoDetect.Visibility.NONE)
-@DataDefinition
-public class TableData implements Serializable, WithLayout {
-
-
- public static final class Builder {
-
- private LayoutEntity layoutEntity;
- private PartitionType partitionType;
- private String database;
- private String table;
-
- public Builder setLayoutEntity(LayoutEntity layoutEntity) {
- this.layoutEntity = layoutEntity;
- return this;
- }
-
- public Builder setPartitionType(PartitionType partitionType) {
- this.partitionType = partitionType;
- return this;
- }
-
- public Builder setDatabase(String database) {
- this.database = database;
- return this;
- }
-
- public Builder setTable(String table) {
- this.table = table;
- return this;
- }
-
- public TableData build() {
- TableData tableData = new TableData();
- tableData.layoutID = layoutEntity.getId();
- tableData.partitionType = this.partitionType;
- tableData.table = this.table;
- tableData.database = this.database;
- return tableData;
- }
- }
- public static Builder builder() {
- return new Builder();
- }
-
- @JsonBackReference
- private TableFlow tableFlow;
-
- @JsonProperty("layout_id")
- private long layoutID;
-
- @JsonProperty("database")
- private String database;
-
- @JsonProperty("table")
- private String table;
-
- @JsonProperty("partition_type")
- private PartitionType partitionType;
-
- @JsonProperty("partitions")
- private final List<TablePartition> partitions = Lists.newArrayList();
-
- private Set<String> allSegmentIds;
-
- public String getDatabase() {
- return database;
- }
-
- public String getTable() {
- return table;
- }
-
- @Override
- public long getLayoutID() {
- return layoutID;
- }
-
- public PartitionType getPartitionType() {
- return partitionType;
- }
-
- protected void checkIsNotCachedAndShared() {
- if (tableFlow != null)
- tableFlow.checkIsNotCachedAndShared();
- }
-
- // create
- public void addPartition(TablePartition partition) {
- Preconditions.checkArgument(partition != null);
- checkIsNotCachedAndShared();
- partitions.removeIf(p ->
- isSameGroup(p.getShardNodes(), partition.getShardNodes()) && p.getSegmentId().equals(partition.getSegmentId())
- );
- partitions.add(partition);
- }
-
- // read
- public List<TablePartition> getPartitions() {
- return Collections.unmodifiableList(partitions);
- }
-
- // update & delete
- public void removePartitions(Predicate<? super TablePartition> filter) {
- checkIsNotCachedAndShared();
- partitions.removeIf(filter);
- }
-
- public void mergePartitions(Set<String> oldSegIds, String mergeSegId) {
- checkIsNotCachedAndShared();
- List<Set<String>> nodeGroups = new ArrayList<>();
- Map<String, Long> sizeInNode = new HashMap<>();
- Map<String, List<SegmentFileStatus>> nodeFileMap = new HashMap<>();
- getPartitions().stream().filter(partition -> oldSegIds.contains(partition.getSegmentId()))
- .forEach(partition -> {
- partition.getSizeInNode().forEach((key, value) ->
- sizeInNode.put(key, sizeInNode.getOrDefault(key, 0L) + value));
-
- partition.getNodeFileMap().forEach((key, value) ->
- nodeFileMap.computeIfAbsent(key, k -> new ArrayList<>()).addAll(value));
-
- for (Set<String> nodeGroup : nodeGroups) {
- if (isSameGroup(nodeGroup, partition.getShardNodes())) {
- nodeGroup.addAll(partition.getShardNodes());
- return;
- }
- }
- nodeGroups.add(new HashSet<>(partition.getShardNodes()));
- });
- List<TablePartition> mergedPartitions = nodeGroups.stream().map(nodeGroup ->
- TablePartition.builder()
- .setSizeInNode(nodeGroup.stream().collect(
- Collectors.toMap(node -> node, node -> sizeInNode.getOrDefault(node, 0L))))
- .setSegmentId(mergeSegId)
- .setShardNodes(Lists.newArrayList(nodeGroup))
- .setId(RandomUtil.randomUUIDStr())
- .setNodeFileMap(nodeGroup.stream().collect(
- Collectors.toMap(node -> node, node -> nodeFileMap.getOrDefault(node, Collections.emptyList()))
- )).build()
- ).collect(Collectors.toList());
- removePartitions(tablePartition -> oldSegIds.contains(tablePartition.getSegmentId()));
- this.partitions.addAll(mergedPartitions);
- }
-
- // utility
- private boolean isSameGroup(Collection<String> a, Collection<String> b) {
- Set<String> aSet = Sets.newHashSet(a);
- Set<String> bSet = Sets.newHashSet(b);
- return aSet.containsAll(bSet) || bSet.containsAll(aSet);
- }
-
- public String getShardJDBCURLs(String project, Set<String> allSegIds) {
- if (partitions.isEmpty()) {
- return null;
- }
- List<Set<String>> aliveShardReplica = SecondStorageQueryRouteUtil.getUsedShard(partitions, project, allSegIds);
- QueryContext queryContext = QueryContext.current();
- List<String> jdbcUrls = SecondStorageNodeHelper.resolveShardToJDBC(aliveShardReplica, queryContext);
-
- if (!CollectionUtils.isEmpty(queryContext.getSecondStorageUrls())) {
- QueryContext.current().setRetrySecondStorage(false);
- }
- QueryContext.current().setSecondStorageUrls(jdbcUrls);
- return ShardOptions$.MODULE$.buildSharding(JavaConverters.asScalaBuffer(jdbcUrls));
- }
-
- public String getSchemaURL() {
- if (partitions.isEmpty()) {
- return null;
- }
-
- return QueryContext.current().getSecondStorageUrls().get(0);
- }
-
- public int getSchemaURLSize() {
- if (partitions.isEmpty()) {
- return 0;
- }
-
- return QueryContext.current().getSecondStorageUrls().size();
- }
-
- public boolean containSegments(Set<String> segmentIds) {
- if (allSegmentIds == null) {
- allSegmentIds = partitions.stream().map(TablePartition::getSegmentId).collect(Collectors.toSet());
- }
- return allSegmentIds.containsAll(segmentIds);
- }
-
- public Set<String> getAllSegments() {
- if (allSegmentIds == null) {
- allSegmentIds = partitions.stream().map(TablePartition::getSegmentId).collect(Collectors.toSet());
- }
-
- return Collections.unmodifiableSet(allSegmentIds);
- }
-
- public void removeNodes(List<String> nodeNames) {
- if (CollectionUtils.isEmpty(nodeNames)) {
- return;
- }
-
- List<TablePartition> newPartitionList = getPartitions().stream().map(partition -> {
- Map<String, Long> sizeInNode = new HashMap<>(partition.getSizeInNode());
- Map<String, List<SegmentFileStatus>> nodeFileMap = new HashMap<>(partition.getNodeFileMap());
- List<String> shardNodes = new ArrayList<>(partition.getShardNodes());
-
- nodeNames.forEach(nodeName -> {
- long size = partition.getSizeInNode().getOrDefault(nodeName, 0L);
- if (size != 0) {
- throw new KylinException(ServerErrorCode.SECOND_STORAGE_DELETE_NODE_FAILED,
- String.format(Locale.ROOT, MsgPicker.getMsg().getSecondStorageDeleteNodeFailed(), nodeName, size));
- }
-
- sizeInNode.remove(nodeName);
- nodeFileMap.remove(nodeName);
- });
-
- shardNodes.removeAll(nodeNames);
-
- return new TablePartition.Builder()
- .setId(partition.getId())
- .setSegmentId(partition.getSegmentId())
- .setShardNodes(shardNodes)
- .setSizeInNode(sizeInNode)
- .setNodeFileMap(nodeFileMap)
- .setSecondaryIndexColumns(partition.getSecondaryIndexColumns())
- .build();
- }).collect(Collectors.toList());
-
- newPartitionList.forEach(this::addPartition);
- }
-
- public void updateSecondaryIndex(Set<Integer> addColumns, Set<Integer> removeColumns) {
- for (TablePartition partition : this.partitions) {
- if (!CollectionUtils.isEmpty(addColumns)) {
- partition.addSecondaryIndex(addColumns);
- }
- if (!CollectionUtils.isEmpty(removeColumns)) {
- partition.removeSecondaryIndex(removeColumns);
- }
- }
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableEntity.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableEntity.java
deleted file mode 100644
index 4374cfb7c2..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableEntity.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.metadata.cube.model.LayoutEntity;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import io.kyligence.kap.secondstorage.metadata.annotation.TableDefinition;
-
-@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-@TableDefinition
-public class TableEntity implements Serializable, WithLayout {
-
- public static final int DEFAULT_SHARD = 100;
-
- public static final class Builder {
-
- private LayoutEntity layoutEntity;
- private List<Integer> primaryIndexColumns = Lists.newArrayList();
- private Set<Integer> secondaryIndexColumns = Sets.newHashSet();
-
- public Builder setLayoutEntity(LayoutEntity layoutEntity) {
- this.layoutEntity = layoutEntity;
- return this;
- }
-
- public Builder setPrimaryIndexColumns(List<Integer> primaryIndexColumns) {
- this.primaryIndexColumns = primaryIndexColumns;
- return this;
- }
-
- public Builder setSecondaryIndexColumns(Set<Integer> secondaryIndexColumns) {
- this.secondaryIndexColumns = secondaryIndexColumns;
- return this;
- }
-
- public TableEntity build() {
- TableEntity table = new TableEntity();
- table.layoutID = layoutEntity.getId();
- if (primaryIndexColumns != null) {
- table.primaryIndexColumns = primaryIndexColumns;
- table.primaryIndexLastModified = System.currentTimeMillis();
- }
- if (secondaryIndexColumns != null) {
- table.secondaryIndexColumns = secondaryIndexColumns;
- table.secondaryIndexLastModified = System.currentTimeMillis();
- }
- return table;
- }
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- @JsonBackReference
- private TablePlan tablePlan;
-
- @JsonProperty("layout_id")
- private long layoutID;
-
- @JsonProperty("shard_numbers")
- private int shardNumbers = DEFAULT_SHARD;
-
- @JsonProperty("primary_index_columns")
- private List<Integer> primaryIndexColumns = Lists.newArrayList();
-
- @JsonProperty("primary_index_last_modified")
- private long primaryIndexLastModified;
-
- @JsonProperty("secondary_index_columns")
- private Set<Integer> secondaryIndexColumns = Sets.newHashSet();
-
- @JsonProperty("secondary_index_last_modified")
- private long secondaryIndexLastModified;
-
- public void checkIsNotCachedAndShared() {
- if (tablePlan != null)
- tablePlan.checkIsNotCachedAndShared();
- }
-
- public void setShardNumbers(int shardNumbers) {
- checkIsNotCachedAndShared();
- this.shardNumbers = shardNumbers;
- }
-
- public void setPrimaryIndexColumns(List<Integer> primaryIndexColumns) {
- checkIsNotCachedAndShared();
- this.primaryIndexColumns = primaryIndexColumns;
- this.primaryIndexLastModified = System.currentTimeMillis();
- }
-
- public void setSecondaryIndexColumns(Set<Integer> secondaryIndexColumns) {
- checkIsNotCachedAndShared();
- this.secondaryIndexColumns = secondaryIndexColumns;
- this.secondaryIndexLastModified = System.currentTimeMillis();
- }
-
- public TablePlan getTablePlan() {
- return tablePlan;
- }
-
- public long getLayoutID() {
- return layoutID;
- }
-
- public int getShardNumbers() {
- return shardNumbers;
- }
-
- public List<Integer> getPrimaryIndexColumns() {
- return this.primaryIndexColumns;
- }
-
- public Set<Integer> getSecondaryIndexColumns() {
- return this.secondaryIndexColumns;
- }
-
- public long getPrimaryIndexLastModified() {
- return this.primaryIndexLastModified;
- }
-
- public long getSecondaryIndexLastModified() {
- return this.secondaryIndexLastModified;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableFlow.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableFlow.java
deleted file mode 100644
index 908f591e13..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TableFlow.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.function.Consumer;
-import java.util.function.Predicate;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.cube.model.LayoutEntity;
-
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
-import io.kyligence.kap.secondstorage.metadata.annotation.DataDefinition;
-
-// CALL FROM CORE
-@DataDefinition
-public class TableFlow extends RootPersistentEntity
- implements Serializable,
- HasLayoutElement<TableData>,
- IManagerAware<TableFlow> {
-
- public static final class Builder {
- private String model;
-
- public Builder setModel(String model) {
- this.model = model;
- return this;
- }
-
- public Builder setDescription(String description) {
- this.description = description;
- return this;
- }
-
- private String description;
-
- public TableFlow build() {
- TableFlow result = new TableFlow();
- result.setUuid(model);
- result.setDescription(description);
- return result;
- }
- }
-
- public static Builder builder() {
- return new Builder();
- }
-
- public static final String TABLEFLOW_RESOURCE_ROOT = "/clickhouse_data";
-
- protected transient Manager<TableFlow> manager;
- @Override
- public void setManager(Manager<TableFlow> manager) {
- this.manager = manager;
- }
-
- @Override
- public void verify() {
- // Here we check everything is ok
- }
-
- @JsonProperty("description")
- private String description;
-
- @JsonManagedReference
- @JsonProperty("data_list")
- private final List<TableData> tableDataList = Lists.newArrayList();
-
- public void setDescription(String description) {
- checkIsNotCachedAndShared();
- this.description = description;
- }
-
- public String getDescription() {
- return description;
- }
-
- public List<TableData> getTableDataList() {
- return Collections.unmodifiableList(tableDataList);
- }
-
- @Override
- public List<TableData> all() {
- return tableDataList;
- }
-
- public void upsertTableData(LayoutEntity layoutEntity, Consumer<TableData> updater, Supplier<TableData> creator) {
- checkIsNotCachedAndShared();
- TableData data = getEntity(layoutEntity)
- .map(tableData -> {
- Preconditions.checkArgument(containIndex(layoutEntity, true));
- updater.accept(tableData);
- return tableData;})
- .orElseGet(() -> {
- TableData newData = creator.get();
- tableDataList.add(newData);
- updater.accept(newData);
- return newData;});
- Preconditions.checkArgument(HasLayoutElement.sameLayout(data, layoutEntity));
- }
-
- public void cleanTableData(Predicate<? super TableData> filter) {
- if (filter == null) {
- return;
- }
-
- checkIsNotCachedAndShared();
- this.tableDataList.removeIf(filter);
- }
-
- public void cleanTableData() {
- checkIsNotCachedAndShared();
- this.tableDataList.clear();
- }
-
- public void removeNodes(List<String> nodeNames) {
- if (CollectionUtils.isEmpty(nodeNames)) {
- return;
- }
-
- checkIsNotCachedAndShared();
- this.tableDataList.forEach(tableData -> tableData.removeNodes(nodeNames));
- }
-
- public Set<Long> getLayoutBySegment(String segmentId) {
- return getTableDataList().stream()
- .filter(tableData -> tableData.containSegments(Collections.singleton(segmentId)))
- .map(TableData::getLayoutID)
- .collect(Collectors.toSet());
- }
-
- public boolean containsLayout(long layoutId) {
- return getTableDataList().stream()
- .anyMatch(tableData -> tableData.getLayoutID() == layoutId);
- }
-
- public List<TableData> getTableData(long layoutId) {
- return getTableDataList().stream().filter(tableData -> tableData.getLayoutID() == layoutId)
- .collect(Collectors.toList());
- }
-
- public void updateSecondaryIndex(long layoutId, Set<Integer> addColumns, Set<Integer> removeColumns) {
- this.tableDataList.forEach(tableData -> {
- if (tableData.getLayoutID() == layoutId) {
- tableData.updateSecondaryIndex(addColumns, removeColumns);
- }
- });
- }
-
- @Override
- public String getResourcePath() {
- return concatResourcePath(getUuid(), manager.project);
- }
-
- public static String concatResourcePath(String name, String project) {
- return new StringBuilder().append("/").append(project).append(TABLEFLOW_RESOURCE_ROOT)
- .append("/").append(name).append(MetadataConstants.FILE_SURFIX).toString();
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof TableFlow)) return false;
- if (!super.equals(o)) return false;
-
- TableFlow tableFlow = (TableFlow) o;
-
- if (!Objects.equals(description, tableFlow.description))
- return false;
- return Objects.equals(tableDataList, tableFlow.tableDataList);
- }
-
- @Override
- public int hashCode() {
- int result = super.hashCode();
- result = 31 * result + (description != null ? description.hashCode() : 0);
- result = 31 * result + tableDataList.hashCode();
- return result;
- }
-
- public TableFlow update(Consumer<TableFlow> updater) {
- Preconditions.checkArgument(manager != null);
- return manager.update(uuid, updater);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePartition.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePartition.java
deleted file mode 100644
index 6f4d4b1e94..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePartition.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import io.kyligence.kap.secondstorage.metadata.annotation.DataDefinition;
-
-// CALL FROM CORE
-@JsonAutoDetect(
- fieldVisibility = JsonAutoDetect.Visibility.NONE,
- getterVisibility = JsonAutoDetect.Visibility.NONE,
- isGetterVisibility = JsonAutoDetect.Visibility.NONE,
- setterVisibility = JsonAutoDetect.Visibility.NONE)
-@DataDefinition
-public class TablePartition implements Serializable {
-
- @JsonProperty("shard_nodes")
- private final List<String> shardNodes = Lists.newArrayList();
- @JsonProperty("size_in_node")
- private Map<String, Long> sizeInNode;
- @JsonProperty("segment_id")
- private String segmentId;
- @JsonProperty("id")
- private String id;
- @JsonProperty("node_file_map")
- private Map<String, List<SegmentFileStatus>> nodeFileMap;
- @JsonProperty("secondary_index_columns")
- private Set<Integer> secondaryIndexColumns = Sets.newHashSet();
-
- public static Builder builder() {
- return new Builder();
- }
-
- public Map<String, Long> getSizeInNode() {
- return sizeInNode== null ? Collections.emptyMap() : sizeInNode;
- }
-
- public List<String> getShardNodes() {
- return Collections.unmodifiableList(shardNodes);
- }
-
- public static final class Builder {
- private List<String> shardNodes;
- private String segmentId;
- private String id;
- private Map<String, Long> sizeInNode;
- private Map<String, List<SegmentFileStatus>> nodeFileMap;
- private Set<Integer> secondaryIndexColumns = Sets.newHashSet();
-
- public Builder setSegmentId(String segmentId) {
- this.segmentId = segmentId;
- return this;
- }
- public Builder setShardNodes(List<String> shardNodes) {
- this.shardNodes = shardNodes;
- return this;
- }
-
- public Builder setId(String id) {
- this.id = id;
- return this;
- }
-
- public Builder setNodeFileMap(Map<String, List<SegmentFileStatus>> nodeFileMap) {
- this.nodeFileMap = nodeFileMap;
- return this;
- }
-
- public Builder setSizeInNode(Map<String, Long> sizeInNode) {
- this.sizeInNode = sizeInNode;
- return this;
- }
-
- public Builder setSecondaryIndexColumns(Set<Integer> secondaryIndexColumns) {
- this.secondaryIndexColumns = secondaryIndexColumns;
- return this;
- }
-
- public TablePartition build() {
- TablePartition partition = new TablePartition();
- partition.shardNodes.addAll(shardNodes);
- partition.segmentId = segmentId;
- partition.id = id;
- partition.nodeFileMap = nodeFileMap;
- partition.sizeInNode = sizeInNode;
- partition.secondaryIndexColumns = secondaryIndexColumns;
- return partition;
- }
- }
- public String getSegmentId() {
- return segmentId;
- }
-
- public String getId() {
- return id;
- }
-
- public Map<String, List<SegmentFileStatus>> getNodeFileMap() {
- return nodeFileMap == null ? Collections.emptyMap() : nodeFileMap;
- }
-
- public void removeSecondaryIndex(Set<Integer> cols) {
- this.secondaryIndexColumns.removeAll(cols);
- }
-
- public Set<Integer> getSecondaryIndexColumns() {
- return this.secondaryIndexColumns;
- }
-
- public void addSecondaryIndex(Set<Integer> cols) {
- this.secondaryIndexColumns.addAll(cols);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePlan.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePlan.java
deleted file mode 100644
index c6eb1838a6..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/TablePlan.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.function.Consumer;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.metadata.cube.model.LayoutEntity;
-
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Lists;
-
-import io.kyligence.kap.secondstorage.metadata.annotation.TableDefinition;
-
-@TableDefinition
-public class TablePlan extends RootPersistentEntity
- implements Serializable,
- HasLayoutElement<TableEntity>,
- IManagerAware<TablePlan> {
-
- public static final class Builder {
- private String model;
-
- public Builder setModel(String model) {
- this.model = model;
- return this;
- }
-
- public Builder setDescription(String description) {
- this.description = description;
- return this;
- }
-
- private String description;
-
- public TablePlan build() {
- TablePlan result = new TablePlan();
- result.setUuid(model);
- result.setDescription(description);
- return result;
- }
- }
- public static Builder builder() {
- return new Builder();
- }
-
- protected transient Manager<TablePlan> manager;
-
- @Override
- public void setManager(Manager<TablePlan> manager) {
- this.manager = manager;
- }
-
- @Override
- public void verify() {
- // Here we check everything is ok
- }
-
- @JsonProperty("description")
- private String description;
-
- @JsonManagedReference
- @JsonProperty("table_metas")
- private final List<TableEntity> tableMetas = Lists.newArrayList();
-
- public List<TableEntity> getTableMetas() {
- return Collections.unmodifiableList(tableMetas);
- }
-
- @Override
- public List<TableEntity> all() {
- return tableMetas;
- }
-
- void addTable(TableEntity entity) {
- checkIsNotCachedAndShared();
- tableMetas.add(entity);
- }
-
- public void setDescription(String description) {
- checkIsNotCachedAndShared();
- this.description = description;
- }
-
- public void cleanTable() {
- checkIsNotCachedAndShared();
- this.tableMetas.clear();
- }
-
- public void cleanTable(Set<Long> layoutIds) {
- if (layoutIds== null || layoutIds.isEmpty()) {
- return;
- }
-
- checkIsNotCachedAndShared();
- this.tableMetas.removeIf(tableEntity -> layoutIds.contains(tableEntity.getLayoutID()));
- }
-
- public void updatePrimaryIndexColumns(long layoutId, List<Integer> primaryIndexColumns) {
- checkIsNotCachedAndShared();
- TableEntity entity = this.tableMetas.stream().filter(tableEntity -> tableEntity.getLayoutID() == layoutId)
- .findFirst().orElse(null);
- if (entity == null) {
- return;
- }
-
- entity.setPrimaryIndexColumns(primaryIndexColumns);
- }
-
- public void updateSecondaryIndexColumns(long layoutId, Set<Integer> secondaryIndexColumns) {
- checkIsNotCachedAndShared();
- TableEntity entity = this.tableMetas.stream().filter(tableEntity -> tableEntity.getLayoutID() == layoutId)
- .findFirst().orElse(null);
- if (entity == null) {
- return;
- }
- entity.setSecondaryIndexColumns(secondaryIndexColumns);
- }
-
- public String getDescription() {
- return description;
- }
-
- @Override
- public String getResourcePath() {
- return manager.getResourcePath(resourceName());
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof TablePlan)) return false;
- if (!super.equals(o)) return false;
-
- TablePlan tablePlan = (TablePlan) o;
-
- if (!Objects.equals(description, tablePlan.description))
- return false;
- return Objects.equals(tableMetas, tablePlan.tableMetas);
- }
-
- @Override
- public int hashCode() {
- int result = super.hashCode();
- result = 31 * result + (description != null ? description.hashCode() : 0);
- result = 31 * result + tableMetas.hashCode();
- return result;
- }
-
- // update
- public TablePlan createTableEntityIfNotExists(LayoutEntity layoutEntity, boolean throwOnDifferentLayout) {
- Preconditions.checkArgument(manager != null);
- if (containIndex(layoutEntity, throwOnDifferentLayout))
- return this;
-
- Optional<TableEntity> preTableEntity = getTableMetas().stream()
- .max(Comparator.comparing(TableEntity::getLayoutID));
-
- TableEntity entity = TableEntity.builder().setLayoutEntity(layoutEntity)
- .setPrimaryIndexColumns(preTableEntity.map(TableEntity::getPrimaryIndexColumns).orElse(null))
- .setSecondaryIndexColumns(preTableEntity.map(TableEntity::getSecondaryIndexColumns).orElse(null))
- .build();
- return createTableEntity(entity);
- }
-
- public TablePlan createTableEntity(TableEntity tableEntity) {
- Preconditions.checkArgument(manager != null);
- return manager.update(uuid, copyForWrite -> copyForWrite.addTable(tableEntity));
- }
-
- public TablePlan update(Consumer<TablePlan> updater) {
- Preconditions.checkArgument(manager != null);
- return manager.update(uuid, updater);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/WithLayout.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/WithLayout.java
deleted file mode 100644
index 9aad928b84..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/WithLayout.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata;
-
-public interface WithLayout {
- long getLayoutID();
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/annotation/DataDefinition.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/annotation/DataDefinition.java
deleted file mode 100644
index 55c96451df..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/annotation/DataDefinition.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata.annotation;
-
-public @interface DataDefinition {
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/annotation/TableDefinition.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/annotation/TableDefinition.java
deleted file mode 100644
index da4a3bddab..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/metadata/annotation/TableDefinition.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.metadata.annotation;
-
-import java.lang.annotation.Documented;
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Documented
-@Target(ElementType.TYPE)
-@Retention(RetentionPolicy.CLASS)
-public @interface TableDefinition {
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/SecondStorageInfo.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/SecondStorageInfo.java
deleted file mode 100644
index 96ff5ba2e7..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/SecondStorageInfo.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.response;
-
-
-import java.util.List;
-import java.util.Map;
-
-// CALL FROM CORE
-public class SecondStorageInfo {
- private long secondStorageSize;
- private Map<String, List<SecondStorageNode>> secondStorageNodes;
- private boolean secondStorageEnabled;
-
- public long getSecondStorageSize() {
- return secondStorageSize;
- }
-
- public SecondStorageInfo setSecondStorageSize(long secondStorageSize) {
- this.secondStorageSize = secondStorageSize;
- return this;
- }
-
- public Map<String, List<SecondStorageNode>> getSecondStorageNodes() {
- return secondStorageNodes;
- }
-
- public SecondStorageInfo setSecondStorageNodes(Map<String, List<SecondStorageNode>> secondStorageNodes) {
- this.secondStorageNodes = secondStorageNodes;
- return this;
- }
-
- public boolean isSecondStorageEnabled() {
- return secondStorageEnabled;
- }
-
- public SecondStorageInfo setSecondStorageEnabled(boolean secondStorageEnabled) {
- this.secondStorageEnabled = secondStorageEnabled;
- return this;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/SecondStorageNode.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/SecondStorageNode.java
deleted file mode 100644
index 25aa1eea07..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/SecondStorageNode.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.response;
-
-import java.io.Serializable;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import io.kyligence.kap.secondstorage.config.Node;
-
-// CALL FROM CORE
-public class SecondStorageNode implements Serializable {
- private static final long serialVersionUID = 0L;
-
- @JsonProperty("name")
- private String name;
- @JsonProperty("ip")
- private String ip;
- @JsonProperty("port")
- private long port;
-
- public SecondStorageNode(Node node) {
- this.ip = node.getIp();
- this.port = node.getPort();
- this.name = node.getName();
- }
-
- public String getName() {
- return name;
- }
-
- public SecondStorageNode setName(String name) {
- this.name = name;
- return this;
- }
-
- public String getIp() {
- return ip;
- }
-
- public SecondStorageNode setIp(String ip) {
- this.ip = ip;
- return this;
- }
-
- public long getPort() {
- return port;
- }
-
- public SecondStorageNode setPort(long port) {
- this.port = port;
- return this;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/SizeInNodeResponse.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/SizeInNodeResponse.java
deleted file mode 100644
index a64d056998..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/SizeInNodeResponse.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.response;
-
-import java.io.Serializable;
-
-public class SizeInNodeResponse implements Serializable {
- private static final long serialVersionUID = 6514965932780603956L;
-
- private String project;
-
- public SizeInNodeResponse(String project) {
- this.project = project;
- }
-
- public String getProject() {
- return project;
- }
-
- public void setProject(String project) {
- this.project = project;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/TableSyncResponse.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/TableSyncResponse.java
deleted file mode 100644
index 3aebc14344..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/response/TableSyncResponse.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.response;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class TableSyncResponse implements Serializable {
- private static final long serialVersionUID = 4072081451417772025L;
- private String project;
- private List<String> nodes;
- private String database;
- private List<String> tables;
-
- public TableSyncResponse(String project) {
- this.project = project;
- }
-
- public TableSyncResponse(String project, List<String> nodes, String database, List<String> tables) {
- this.project = project;
- this.nodes = nodes;
- this.database = database;
- this.tables = tables;
- }
-
- public String getProject() {
- return project;
- }
-
- public void setProject(String project) {
- this.project = project;
- }
-
- public List<String> getNodes() {
- return nodes;
- }
-
- public void setNodes(List<String> nodes) {
- this.nodes = nodes;
- }
-
- public String getDatabase() {
- return database;
- }
-
- public void setDatabase(String database) {
- this.database = database;
- }
-
- public List<String> getTables() {
- return tables;
- }
-
- public void setTables(List<String> tables) {
- this.tables = tables;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/ConvertUtils.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/ConvertUtils.java
deleted file mode 100644
index b7461e58ac..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/ConvertUtils.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.util;
-
-import java.util.Locale;
-
-import io.kyligence.kap.secondstorage.config.SecondStorageProjectModelSegment;
-
-public class ConvertUtils {
- public static <T> T convertValue(Object rawValue, Class<?> clazz) {
- if (Integer.class.equals(clazz)) {
- return (T) convertToInt(rawValue);
- } else if (Long.class.equals(clazz)) {
- return (T) convertToLong(rawValue);
- } else if (Boolean.class.equals(clazz)) {
- return (T) convertToBoolean(rawValue);
- } else if (Float.class.equals(clazz)) {
- return (T) convertToFloat(rawValue);
- } else if (Double.class.equals(clazz)) {
- return (T) convertToDouble(rawValue);
- } else if (String.class.equals(clazz)) {
- return (T) convertToString(rawValue);
- } else if (SecondStorageProjectModelSegment.class.equals(clazz)) {
- return (T) rawValue;
- }
- throw new IllegalArgumentException("Unsupported type: " + clazz);
- }
-
- static String convertToString(Object o) {
- if (o.getClass() == String.class) {
- return (String) o;
- }
- return o.toString();
- }
-
- static Integer convertToInt(Object o) {
- if (o.getClass() == Integer.class) {
- return (Integer) o;
- } else if (o.getClass() == Long.class) {
- long value = (Long) o;
- if (value <= Integer.MAX_VALUE && value >= Integer.MIN_VALUE) {
- return (int) value;
- } else {
- throw new IllegalArgumentException(String.format(Locale.ROOT,
- "Configuration value %s overflow/underflow the integer type.", value));
- }
- }
- return Integer.parseInt(o.toString());
- }
-
- static Long convertToLong(Object o) {
- if (o.getClass() == Long.class) {
- return (Long) o;
- } else if (o.getClass() == Integer.class) {
- return ((Integer) o).longValue();
- }
- return Long.parseLong(o.toString());
- }
-
- static Boolean convertToBoolean(Object o) {
- if (o.getClass() == Boolean.class) {
- return (Boolean) o;
- }
-
- switch (o.toString().toUpperCase(Locale.ROOT)) {
- case "TRUE":
- return true;
- case "FALSE":
- return false;
- default:
- throw new IllegalArgumentException(String.format(Locale.ROOT,
- "Unrecognized option for boolean: %s. Expected either true or false(case insensitive)", o));
- }
- }
-
- static Float convertToFloat(Object o) {
- if (o.getClass() == Float.class) {
- return (Float) o;
- } else if (o.getClass() == Double.class) {
- double value = ((Double) o);
- if (value == 0.0 || (value >= Float.MIN_VALUE && value <= Float.MAX_VALUE)
- || (value >= -Float.MAX_VALUE && value <= -Float.MIN_VALUE)) {
- return (float) value;
- } else {
- throw new IllegalArgumentException(String.format(Locale.ROOT,
- "Configuration value %s overflows/underflow the float type.", value));
- }
- }
-
- return Float.parseFloat(o.toString());
- }
-
- static Double convertToDouble(Object o) {
- if (o.getClass() == Double.class) {
- return (Double) o;
- } else if (o.getClass() == Float.class) {
- return ((Float) o).doubleValue();
- }
-
- return Double.parseDouble(o.toString());
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/SecondStorageDateUtils.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/SecondStorageDateUtils.java
deleted file mode 100644
index 2dd3814c20..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/SecondStorageDateUtils.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.util;
-
-import java.sql.Date;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-import org.apache.kylin.metadata.model.SegmentRange;
-
-public class SecondStorageDateUtils {
-
- public static boolean isInfinite(long start, long end) {
- return start == 0 && end == Long.MAX_VALUE;
- }
-
- /**
- * @param start Include
- * @param end exclude
- * @return
- */
- public static List<String> splitByDayStr(long start, long end) {
- return splitByDay(start, end).stream().map(Objects::toString).collect(Collectors.toList());
- }
-
- public static List<Date> splitByDay(SegmentRange<Long> range) {
- return splitByDay(range.getStart(), range.getEnd());
- }
-
- public static List<Date> splitByDay(long start, long end) {
- if (end == Long.MAX_VALUE) {
- throw new IllegalArgumentException("segmentRange end is invalid.");
- }
- List<Date> partitions = new ArrayList<>();
- while (start < end) {
- partitions.add(new Date(start));
- start = start + 24 * 60 * 60 * 1000;
- }
- return partitions;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/SecondStorageJobUtil.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/SecondStorageJobUtil.java
deleted file mode 100644
index 311ec3a8af..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/SecondStorageJobUtil.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage.util;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.exception.JobErrorCode;
-import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.msg.MsgPicker;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableManager;
-import org.apache.kylin.job.execution.JobTypeEnum;
-
-import io.kyligence.kap.secondstorage.SecondStorageUtil;
-
-// CALL FROM CORE
-public class SecondStorageJobUtil {
- public static final Set<JobTypeEnum> SECOND_STORAGE_JOBS = Sets.newHashSet(JobTypeEnum.EXPORT_TO_SECOND_STORAGE,
- JobTypeEnum.SECOND_STORAGE_SEGMENT_CLEAN, JobTypeEnum.SECOND_STORAGE_MODEL_CLEAN);
-
- public static List<AbstractExecutable> findSecondStorageJobByProject(String project) {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- ExecutableManager executableManager = ExecutableManager.getInstance(config, project);
- return executableManager.getJobs().stream().map(executableManager::getJob)
- .filter(job -> SECOND_STORAGE_JOBS.contains(job.getJobType())).collect(Collectors.toList());
- }
-
- public static void validateSegment(String project, String modelId, List<String> segmentIds) {
- List<AbstractExecutable> jobs = findSecondStorageJobByProject(project).stream()
- .filter(job -> SecondStorageUtil.RUNNING_STATE.contains(job.getStatus()))
- .filter(job -> Objects.equals(job.getTargetModelId(), modelId)).filter(job -> {
- List<String> targetSegments = Optional.ofNullable(job.getTargetSegments())
- .orElse(Collections.emptyList());
- return targetSegments.stream().anyMatch(segmentIds::contains);
- }).collect(Collectors.toList());
- if (!jobs.isEmpty()) {
- throw new KylinException(JobErrorCode.SECOND_STORAGE_JOB_EXISTS,
- MsgPicker.getMsg().getSecondStorageConcurrentOperate());
- }
- }
-
- public static void validateModel(String project, String modelId) {
- List<AbstractExecutable> jobs = findSecondStorageJobByProject(project).stream()
- .filter(job -> SecondStorageUtil.RUNNING_STATE.contains(job.getStatus()))
- .filter(job -> Objects.equals(job.getTargetModelId(), modelId)).collect(Collectors.toList());
- if (!jobs.isEmpty()) {
- throw new KylinException(JobErrorCode.SECOND_STORAGE_JOB_EXISTS,
- MsgPicker.getMsg().getSecondStorageConcurrentOperate());
- }
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/SecondStorageSqlUtils.java b/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/SecondStorageSqlUtils.java
deleted file mode 100644
index f77bde5e66..0000000000
--- a/outdated/second-storage/core/src/main/java/io/kyligence/kap/secondstorage/util/SecondStorageSqlUtils.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage.util;
-
-import org.apache.commons.lang3.StringUtils;
-
-public class SecondStorageSqlUtils {
- private static final String IF_NOT_EXISTS = "IF NOT EXISTS";
- public static String addIfNotExists(String sql, String type) {
- if (StringUtils.isBlank(sql)) {
- return sql;
- }
- int index = StringUtils.indexOfIgnoreCase(sql, type);
- if (index < 0) {
- return sql;
- }
-
- return sql.substring(0, index + type.length()) + " "
- + IF_NOT_EXISTS + sql.substring(index + type.length());
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/common/SecondStorageConfig.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/common/SecondStorageConfig.java
deleted file mode 100644
index 01536ad90b..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/common/SecondStorageConfig.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common;
-
-import io.kyligence.kap.secondstorage.SecondStorageConstants;
-
-/**
- * To access protect method of {@link KylinConfig}, we have to put this class in {@link org.apache.kylin.common}
- */
-public class SecondStorageConfig {
-
- private final KylinConfig config;
-
- private SecondStorageConfig(KylinConfig config) {
- this.config = config;
- }
-
- public static SecondStorageConfig getInstanceFromEnv() {
- return wrap(KylinConfig.getInstanceFromEnv());
- }
-
- public static SecondStorageConfig wrap(KylinConfig config) {
- return new SecondStorageConfig(config);
- }
-
- public int getReplicaNum() {
- return Integer.parseInt(config.getOptional(SecondStorageConstants.NODE_REPLICA, "1"));
- }
-
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/ProjectJob.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/ProjectJob.java
deleted file mode 100644
index 0d37525baa..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/ProjectJob.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-public interface ProjectJob {
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageCleanJobBuildParams.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageCleanJobBuildParams.java
deleted file mode 100644
index 94fb0bc5ce..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageCleanJobBuildParams.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.util.Objects;
-import java.util.Set;
-
-import org.apache.kylin.job.execution.JobTypeEnum;
-import org.apache.kylin.job.factory.JobFactory;
-import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.cube.model.NDataSegment;
-
-public class SecondStorageCleanJobBuildParams extends JobFactory.JobBuildParams {
- private String project;
- private String modelId;
- private String dataflowId;
- private Set<Long> secondStorageDeleteLayoutIds;
-
- public SecondStorageCleanJobBuildParams(Set<NDataSegment> segments, JobParam jobParam, JobTypeEnum jobType) {
- super(segments, jobParam.getProcessLayouts(), jobParam.getOwner(), jobType, jobParam.getJobId(), null,
- jobParam.getIgnoredSnapshotTables(), null, null);
- }
-
- public String getProject() {
- return project;
- }
-
- public SecondStorageCleanJobBuildParams setProject(String project) {
- this.project = project;
- return this;
- }
-
- public String getModelId() {
- return modelId;
- }
-
- public SecondStorageCleanJobBuildParams setModelId(String modelId) {
- this.modelId = modelId;
- return this;
- }
-
- public String getDataflowId() {
- return dataflowId;
- }
-
- public SecondStorageCleanJobBuildParams setDataflowId(String dataflowId) {
- this.dataflowId = dataflowId;
- return this;
- }
-
- public void setSecondStorageDeleteLayoutIds(Set<Long> secondStorageDeleteLayoutIds) {
- if (Objects.nonNull(secondStorageDeleteLayoutIds)) {
- this.secondStorageDeleteLayoutIds = secondStorageDeleteLayoutIds;
- }
- }
-
- public Set<Long> getSecondStorageDeleteLayoutIds() {
- return this.secondStorageDeleteLayoutIds;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageCleanJobUtil.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageCleanJobUtil.java
deleted file mode 100644
index de89324712..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageCleanJobUtil.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.common.ExecutableUtil;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.model.JobParam;
-
-
-public class SecondStorageCleanJobUtil extends ExecutableUtil {
-
- public static boolean isProjectCleanJob(AbstractExecutable executable) {
- return executable instanceof ProjectJob;
- }
-
- @Override
- public void computeLayout(JobParam jobParam) {
- // empty
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageJobParamUtil.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageJobParamUtil.java
deleted file mode 100644
index ff538cd3ce..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageJobParamUtil.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.job;
-
-import static org.apache.kylin.job.common.ExecutableUtil.registerImplementation;
-
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
-import org.apache.kylin.job.execution.JobTypeEnum;
-import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.cube.model.LayoutEntity;
-
-public class SecondStorageJobParamUtil {
-
- static {
- registerImplementation(JobTypeEnum.EXPORT_TO_SECOND_STORAGE, new SecondStorageJobUtil());
- registerImplementation(JobTypeEnum.SECOND_STORAGE_MODEL_CLEAN, new SecondStorageCleanJobUtil());
- registerImplementation(JobTypeEnum.SECOND_STORAGE_SEGMENT_CLEAN, new SecondStorageCleanJobUtil());
- registerImplementation(JobTypeEnum.SECOND_STORAGE_NODE_CLEAN, new SecondStorageCleanJobUtil());
- registerImplementation(JobTypeEnum.SECOND_STORAGE_INDEX_CLEAN, new SecondStorageCleanJobUtil());
- registerImplementation(JobTypeEnum.SECOND_STORAGE_REFRESH_SECONDARY_INDEXES,
- new SecondStorageRefreshSecondaryIndexUtil());
- }
-
- private SecondStorageJobParamUtil() {
- throw new IllegalStateException("Utility class");
- }
-
- public static JobParam of(String project, String model, String owner, Stream<String> segmentIDs) {
- final JobParam param = new JobParam(model, owner);
- param.setProject(project);
- param.withTargetSegments(segmentIDs.collect(Collectors.toSet()))
- .withJobTypeEnum(JobTypeEnum.EXPORT_TO_SECOND_STORAGE);
- param.getCondition().put(JobParam.ConditionConstant.REFRESH_ALL_LAYOUTS, Boolean.FALSE);
- return param;
- }
-
- public static JobParam projectCleanParam(String project, String owner) {
- JobParam param = new JobParam("", owner);
- param.setJobTypeEnum(JobTypeEnum.SECOND_STORAGE_NODE_CLEAN);
- param.setProject(project);
- return param;
- }
-
- public static JobParam modelCleanParam(String project, String model, String owner) {
- JobParam param = new JobParam(model, owner);
- param.setProject(project);
- param.setJobTypeEnum(JobTypeEnum.SECOND_STORAGE_MODEL_CLEAN);
- return param;
- }
-
- public static JobParam segmentCleanParam(String project, String model, String owner, Set<String> ids) {
- JobParam param = new JobParam(model, owner);
- param.setProject(project);
- param.withTargetSegments(ids);
- param.setJobTypeEnum(JobTypeEnum.SECOND_STORAGE_SEGMENT_CLEAN);
- return param;
- }
-
- /**
- * build delete layout table parameters
- *
- *
- * @param project project name
- * @param model model id
- * @param owner owner
- * @param needDeleteLayoutIds required delete ids of layout
- * @return job parameters
- */
- public static JobParam layoutCleanParam(String project, String model, String owner, Set<Long> needDeleteLayoutIds,
- Set<String> segmentIds) {
- JobParam param = new JobParam(model, owner);
- param.setProject(project);
- param.withTargetSegments(segmentIds);
- param.setSecondStorageDeleteLayoutIds(needDeleteLayoutIds);
- param.setJobTypeEnum(JobTypeEnum.SECOND_STORAGE_INDEX_CLEAN);
- return param;
- }
-
- public static JobParam refreshSecondaryIndexParam(String project, String model, String owner, LayoutEntity layout,
- Set<Integer> newColumns, Set<Integer> deleteColumns) {
- JobParam param = new JobParam(model, owner);
- param.setProject(project);
- param.setProcessLayouts(Sets.newHashSet(layout));
- param.getCondition().put(JobTypeEnum.SECOND_STORAGE_REFRESH_SECONDARY_INDEXES.name() + "_ADD", newColumns);
- param.getCondition().put(JobTypeEnum.SECOND_STORAGE_REFRESH_SECONDARY_INDEXES.name() + "_DELETE",
- deleteColumns);
- param.setJobTypeEnum(JobTypeEnum.SECOND_STORAGE_REFRESH_SECONDARY_INDEXES);
- return param;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageJobUtil.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageJobUtil.java
deleted file mode 100644
index 7e51f68616..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageJobUtil.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.job;
-
-import static org.apache.kylin.common.exception.ServerErrorCode.FAILED_CREATE_JOB;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.msg.MsgPicker;
-import org.apache.kylin.job.common.ExecutableUtil;
-import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.cube.model.IndexEntity;
-import org.apache.kylin.metadata.cube.model.LayoutEntity;
-import org.apache.kylin.metadata.cube.model.NDataLayout;
-import org.apache.kylin.metadata.cube.model.NDataSegment;
-import org.apache.kylin.metadata.cube.model.NDataflow;
-import org.apache.kylin.metadata.cube.model.NDataflowManager;
-import org.apache.kylin.metadata.model.NDataModel;
-import org.apache.kylin.metadata.model.SegmentRange;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.glassfish.jersey.internal.guava.Sets;
-import org.msgpack.core.Preconditions;
-
-import io.kyligence.kap.secondstorage.SecondStorage;
-import io.kyligence.kap.secondstorage.SecondStorageUtil;
-import io.kyligence.kap.secondstorage.metadata.Manager;
-import io.kyligence.kap.secondstorage.metadata.TableFlow;
-import io.kyligence.kap.secondstorage.metadata.TablePlan;
-
-public class SecondStorageJobUtil extends ExecutableUtil {
-
- private static boolean isValidateSegments(NDataSegment segment, SegmentRange<?> range) {
- return segment.getSegRange().startStartMatch(range) && segment.getSegRange().endEndMatch(range)
- && (segment.getStatus() == SegmentStatusEnum.READY || segment.getStatus() == SegmentStatusEnum.WARNING);
- }
-
- @Override
- public void computeLayout(JobParam jobParam) {
- final KylinConfig config = KylinConfig.getInstanceFromEnv();
- final String project = jobParam.getProject();
- final String model = jobParam.getModel();
- final NDataflow df = NDataflowManager.getInstance(config, project).getDataflow(model);
-
- List<NDataSegment> segments = df.getSegments().stream()
- .filter(segment -> isValidateSegments(segment, segment.getSegRange())).collect(Collectors.toList());
-
- if (segments.isEmpty()) {
- throw new KylinException(FAILED_CREATE_JOB, MsgPicker.getMsg().getAddJobCheckSegmentReadyFail());
- }
-
- Manager<TablePlan> tablePlanManager = SecondStorage.tablePlanManager(config, project);
- Manager<TableFlow> tableFlowManager = SecondStorage.tableFlowManager(config, project);
- TablePlan plan = tablePlanManager.makeSureRootEntity(model);
- tableFlowManager.makeSureRootEntity(model);
- Map<Long, List<LayoutEntity>> layouts = segments.stream()
- .flatMap(nDataSegment -> nDataSegment.getLayoutsMap().values().stream()).map(NDataLayout::getLayout)
- .filter(SecondStorageUtil::isBaseTableIndex).collect(Collectors.groupingBy(LayoutEntity::getIndexId));
-
- NDataModel dataModel = df.getModel();
- if (!segments.get(0).getSegRange().isInfinite()) {
- LayoutEntity baseTableIndex = segments.get(0).getIndexPlan().getBaseTableLayout();
- String partitionCol = dataModel.getPartitionDesc().getPartitionDateColumn();
- Preconditions.checkState(
- baseTableIndex.getColumns().stream().map(TblColRef::getTableDotName)
- .anyMatch(col -> Objects.equals(col, partitionCol)),
- "Table index should contains partition column " + partitionCol);
- }
-
- Set<LayoutEntity> processed = Sets.newHashSet();
- for (Map.Entry<Long, List<LayoutEntity>> entry : layouts.entrySet()) {
- LayoutEntity layoutEntity = entry.getValue().stream()
- .filter(l -> l.isBaseIndex() && IndexEntity.isTableIndex(l.getId())).findFirst().orElse(null);
- if (layoutEntity != null) {
- processed.add(layoutEntity);
- plan = plan.createTableEntityIfNotExists(layoutEntity, true);
- }
- }
- jobParam.setProcessLayouts(new HashSet<>(processed));
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageRefreshSecondaryIndexJobParams.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageRefreshSecondaryIndexJobParams.java
deleted file mode 100644
index be2d5f86be..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageRefreshSecondaryIndexJobParams.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.job;
-
-import java.util.Collections;
-import java.util.Set;
-
-import org.apache.kylin.job.execution.JobTypeEnum;
-import org.apache.kylin.job.factory.JobFactory;
-import org.apache.kylin.job.model.JobParam;
-
-public class SecondStorageRefreshSecondaryIndexJobParams extends JobFactory.JobBuildParams {
- private String project;
- private String modelId;
- private Set<Integer> newIndexes;
- private Set<Integer> toBeDeleteIndexed;
-
- public SecondStorageRefreshSecondaryIndexJobParams(JobParam jobParam, JobTypeEnum jobType) {
- super(Collections.emptySet(), jobParam.getProcessLayouts(), jobParam.getOwner(), jobType, jobParam.getJobId(),
- null, null, null, null);
- }
-
- public String getProject() {
- return project;
- }
-
- public SecondStorageRefreshSecondaryIndexJobParams setProject(String project) {
- this.project = project;
- return this;
- }
-
- public String getModelId() {
- return modelId;
- }
-
- public SecondStorageRefreshSecondaryIndexJobParams setModelId(String modelId) {
- this.modelId = modelId;
- return this;
- }
-
- public void setNewIndexes(Set<Integer> newIndexes) {
- this.newIndexes = newIndexes;
- }
-
- public void setToBeDeleteIndexed(Set<Integer> toBeDeleteIndexed) {
- this.toBeDeleteIndexed = toBeDeleteIndexed;
- }
-
- public Set<Integer> getNewIndexes() {
- return newIndexes;
- }
-
- public Set<Integer> getToBeDeleteIndexed() {
- return toBeDeleteIndexed;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageRefreshSecondaryIndexUtil.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageRefreshSecondaryIndexUtil.java
deleted file mode 100644
index fbe313dcf9..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageRefreshSecondaryIndexUtil.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.job;
-
-import org.apache.kylin.job.common.ExecutableUtil;
-import org.apache.kylin.job.model.JobParam;
-
-public class SecondStorageRefreshSecondaryIndexUtil extends ExecutableUtil {
-
- @Override
- public void computeLayout(JobParam jobParam) {
- // empty
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageStepFactory.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageStepFactory.java
deleted file mode 100644
index e60bba321f..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageStepFactory.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.Consumer;
-import java.util.function.Supplier;
-
-import org.apache.kylin.job.execution.AbstractExecutable;
-
-public class SecondStorageStepFactory {
- private static final Map<Class<? extends SecondStorageStep>, Supplier<AbstractExecutable>> SUPPLIER_MAP //
- = new ConcurrentHashMap<>(5);
-
- public static void register(Class<? extends SecondStorageStep> stepType,
- Supplier<AbstractExecutable> stepSupplier) {
- SUPPLIER_MAP.put(stepType, stepSupplier);
- }
-
- public static AbstractExecutable create(Class<? extends SecondStorageStep> stepType,
- Consumer<AbstractExecutable> paramInjector) {
- AbstractExecutable step = SUPPLIER_MAP.get(stepType).get();
- paramInjector.accept(step);
- return step;
- }
-
- public interface SecondStorageStep {
- }
-
- public interface SecondStorageLoadStep extends SecondStorageStep {
-
- }
-
- public interface SecondStorageRefreshStep extends SecondStorageStep {
- }
-
- public interface SecondStorageMergeStep extends SecondStorageStep {
- }
-
- public interface SecondStorageIndexClean extends SecondStorageStep {
- }
-
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/AbstractSecondStorageJobHanlder.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/AbstractSecondStorageJobHanlder.java
deleted file mode 100644
index eab57c3f75..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/AbstractSecondStorageJobHanlder.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.handler;
-
-import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.job.model.JobParam;
-
-public abstract class AbstractSecondStorageJobHanlder extends AbstractJobHandler {
-
- @Override
- protected void checkBeforeHandle(JobParam jobParam) {
- String project = jobParam.getProject();
- Preconditions.checkNotNull(project);
- }
-
- @Override
- protected boolean needComputeJobBucket() {
- return false;
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageIndexCleanJobHandler.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageIndexCleanJobHandler.java
deleted file mode 100644
index 0371f3efa6..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageIndexCleanJobHandler.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.handler;
-
-import java.util.stream.Collectors;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.job.SecondStorageCleanJobBuildParams;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.JobTypeEnum;
-import org.apache.kylin.job.factory.JobFactory;
-import org.apache.kylin.job.factory.JobFactoryConstant;
-import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.cube.model.NDataflow;
-import org.apache.kylin.metadata.cube.model.NDataflowManager;
-
-public class SecondStorageIndexCleanJobHandler extends AbstractSecondStorageJobHanlder {
- @Override
- protected AbstractExecutable createJob(JobParam jobParam) {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- NDataflow dataflow = NDataflowManager.getInstance(kylinConfig, jobParam.getProject())
- .getDataflow(jobParam.getModel());
- SecondStorageCleanJobBuildParams params = new SecondStorageCleanJobBuildParams(
- jobParam.getTargetSegments().stream().map(dataflow::getSegment).collect(Collectors.toSet()), jobParam,
- JobTypeEnum.SECOND_STORAGE_INDEX_CLEAN);
- params.setProject(dataflow.getProject());
- params.setModelId(dataflow.getModel().getId());
- params.setDataflowId(dataflow.getId());
- params.setSecondStorageDeleteLayoutIds(jobParam.getSecondStorageDeleteLayoutIds());
- return JobFactory.createJob(JobFactoryConstant.STORAGE_INDEX_CLEAN_FACTORY, params);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageModelCleanJobHandler.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageModelCleanJobHandler.java
deleted file mode 100644
index 92cd3d92a3..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageModelCleanJobHandler.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.handler;
-
-import org.apache.kylin.metadata.cube.model.NDataflow;
-import org.apache.kylin.metadata.cube.model.NDataflowManager;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.job.SecondStorageCleanJobBuildParams;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.JobTypeEnum;
-import org.apache.kylin.job.factory.JobFactory;
-import org.apache.kylin.job.factory.JobFactoryConstant;
-import org.apache.kylin.job.model.JobParam;
-
-public class SecondStorageModelCleanJobHandler extends AbstractSecondStorageJobHanlder {
- @Override
- protected AbstractExecutable createJob(JobParam jobParam) {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- NDataflow dataflow = NDataflowManager.getInstance(kylinConfig, jobParam.getProject())
- .getDataflow(jobParam.getModel());
- SecondStorageCleanJobBuildParams params = new SecondStorageCleanJobBuildParams(
- null,
- jobParam,
- JobTypeEnum.SECOND_STORAGE_MODEL_CLEAN);
- params.setProject(dataflow.getProject());
- params.setModelId(dataflow.getModel().getId());
- params.setDataflowId(dataflow.getId());
- return JobFactory.createJob(JobFactoryConstant.STORAGE_MODEL_CLEAN_FACTORY, params);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageProjectCleanJobHandler.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageProjectCleanJobHandler.java
deleted file mode 100644
index 59b9d33dbe..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageProjectCleanJobHandler.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.handler;
-
-import java.util.Collections;
-
-import org.apache.kylin.job.SecondStorageCleanJobBuildParams;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.JobTypeEnum;
-import org.apache.kylin.job.factory.JobFactory;
-import org.apache.kylin.job.factory.JobFactoryConstant;
-import org.apache.kylin.job.model.JobParam;
-
-public class SecondStorageProjectCleanJobHandler extends AbstractSecondStorageJobHanlder {
- @Override
- protected AbstractExecutable createJob(JobParam jobParam) {
- SecondStorageCleanJobBuildParams params = new SecondStorageCleanJobBuildParams(Collections.emptySet(), jobParam,
- JobTypeEnum.SECOND_STORAGE_NODE_CLEAN);
- params.setProject(jobParam.getProject());
- return JobFactory.createJob(JobFactoryConstant.STORAGE_NODE_CLEAN_FACTORY, params);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageRefreshSecondaryIndexJobHandler.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageRefreshSecondaryIndexJobHandler.java
deleted file mode 100644
index 36b2c0ed17..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageRefreshSecondaryIndexJobHandler.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.handler;
-
-import java.util.Set;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.job.SecondStorageRefreshSecondaryIndexJobParams;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.JobTypeEnum;
-import org.apache.kylin.job.factory.JobFactory;
-import org.apache.kylin.job.factory.JobFactoryConstant;
-import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.cube.model.NDataflow;
-import org.apache.kylin.metadata.cube.model.NDataflowManager;
-
-
-public class SecondStorageRefreshSecondaryIndexJobHandler extends AbstractSecondStorageJobHanlder {
- @Override
- protected AbstractExecutable createJob(JobParam jobParam) {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- NDataflow df = NDataflowManager.getInstance(kylinConfig, jobParam.getProject())
- .getDataflow(jobParam.getModel());
- SecondStorageRefreshSecondaryIndexJobParams params = new SecondStorageRefreshSecondaryIndexJobParams(jobParam,
- JobTypeEnum.SECOND_STORAGE_REFRESH_SECONDARY_INDEXES);
- params.setProject(df.getProject());
- params.setModelId(df.getId());
- params.setNewIndexes((Set<Integer>) jobParam.getCondition()
- .get(JobTypeEnum.SECOND_STORAGE_REFRESH_SECONDARY_INDEXES.name() + "_ADD"));
- params.setToBeDeleteIndexed((Set<Integer>) jobParam.getCondition()
- .get(JobTypeEnum.SECOND_STORAGE_REFRESH_SECONDARY_INDEXES.name() + "_DELETE"));
- return JobFactory.createJob(JobFactoryConstant.STORAGE_REFRESH_SECONDARY_INDEXES_FACTORY, params);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageSegmentCleanJobHandler.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageSegmentCleanJobHandler.java
deleted file mode 100644
index 2570717c61..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageSegmentCleanJobHandler.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.handler;
-
-import java.util.stream.Collectors;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.job.SecondStorageCleanJobBuildParams;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.JobTypeEnum;
-import org.apache.kylin.job.factory.JobFactory;
-import org.apache.kylin.job.factory.JobFactoryConstant;
-import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.cube.model.NDataflow;
-import org.apache.kylin.metadata.cube.model.NDataflowManager;
-
-public class SecondStorageSegmentCleanJobHandler extends AbstractSecondStorageJobHanlder {
- @Override
- protected AbstractExecutable createJob(JobParam jobParam) {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- NDataflow dataflow = NDataflowManager.getInstance(kylinConfig, jobParam.getProject())
- .getDataflow(jobParam.getModel());
- SecondStorageCleanJobBuildParams params = new SecondStorageCleanJobBuildParams(
- jobParam.getTargetSegments().stream().map(dataflow::getSegment).collect(Collectors.toSet()), jobParam,
- JobTypeEnum.SECOND_STORAGE_SEGMENT_CLEAN);
- params.setProject(dataflow.getProject());
- params.setModelId(dataflow.getModel().getId());
- params.setDataflowId(dataflow.getId());
- return JobFactory.createJob(JobFactoryConstant.STORAGE_SEGMENT_CLEAN_FACTORY, params);
- }
-}
diff --git a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageSegmentLoadJobHandler.java b/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageSegmentLoadJobHandler.java
deleted file mode 100644
index 715287c287..0000000000
--- a/outdated/second-storage/core/src/main/java/org/apache/kylin/job/handler/SecondStorageSegmentLoadJobHandler.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.handler;
-
-import static org.apache.kylin.common.exception.ServerErrorCode.BASE_TABLE_INDEX_NOT_AVAILABLE;
-import static org.apache.kylin.common.exception.ServerErrorCode.FAILED_CREATE_JOB_EXPORT_TO_TIERED_STORAGE_WITHOUT_BASE_INDEX;
-import static org.apache.kylin.common.exception.ServerErrorCode.SECOND_STORAGE_ADD_JOB_FAILED;
-import static org.apache.kylin.guava30.shaded.common.base.Preconditions.checkNotNull;
-import static org.apache.kylin.job.factory.JobFactoryConstant.STORAGE_JOB_FACTORY;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.msg.MsgPicker;
-import org.apache.kylin.job.exception.JobSubmissionException;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.ExecutableManager;
-import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.job.execution.JobTypeEnum;
-import org.apache.kylin.job.factory.JobFactory;
-import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.cube.model.NDataLayout;
-import org.apache.kylin.metadata.cube.model.NDataSegment;
-import org.apache.kylin.metadata.cube.model.NDataflow;
-import org.apache.kylin.metadata.cube.model.NDataflowManager;
-
-import io.kyligence.kap.secondstorage.SecondStorageUtil;
-
-public class SecondStorageSegmentLoadJobHandler extends AbstractJobHandler {
-
- @Override
- protected AbstractExecutable createJob(JobParam jobParam) {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
-
- NDataflow dataflow = NDataflowManager.getInstance(kylinConfig, jobParam.getProject())
- .getDataflow(jobParam.getModel());
- List<String> hasBaseIndexSegmentIds = jobParam.getTargetSegments().stream().map(dataflow::getSegment)
- .filter(segment -> segment.getLayoutsMap().values().stream().map(NDataLayout::getLayout)
- .anyMatch(SecondStorageUtil::isBaseTableIndex))
- .map(NDataSegment::getId).collect(Collectors.toList());
-
- if (hasBaseIndexSegmentIds.isEmpty()) {
- throw new KylinException(BASE_TABLE_INDEX_NOT_AVAILABLE,
- MsgPicker.getMsg().getSecondStorageSegmentWithoutBaseIndex());
- }
-
- return JobFactory.createJob(STORAGE_JOB_FACTORY, new JobFactory.JobBuildParams(
- jobParam.getTargetSegments().stream().map(dataflow::getSegment).collect(Collectors.toSet()),
- jobParam.getProcessLayouts(), jobParam.getOwner(), JobTypeEnum.EXPORT_TO_SECOND_STORAGE,
- jobParam.getJobId(), null, jobParam.getIgnoredSnapshotTables(), null, null, jobParam.getExtParams()));
- }
-
- @Override
- protected boolean needComputeJobBucket() {
- return false;
- }
-
- @Override
- protected void checkBeforeHandle(JobParam jobParam) {
- String model = jobParam.getModel();
- String project = jobParam.getProject();
- checkNotNull(project);
- checkNotNull(model);
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- ExecutableManager execManager = ExecutableManager.getInstance(kylinConfig, project);
- NDataflowManager dataflowManager = NDataflowManager.getInstance(kylinConfig, project);
- List<AbstractExecutable> executables = execManager.listExecByModelAndStatus(model, ExecutableState::isRunning,
- JobTypeEnum.EXPORT_TO_SECOND_STORAGE);
- NDataflow dataflow = dataflowManager.getDataflow(jobParam.getModel());
-
- Set<String> targetSegs = new HashSet<>(jobParam.getTargetSegments());
- List<String> failedSegs = executables.stream().flatMap(executable -> executable.getTargetSegments().stream())
- .filter(targetSegs::contains).collect(Collectors.toList());
-
- // segment doesn't have base table index
- List<String> noBaseIndexSegs = targetSegs.stream().filter(seg -> {
- NDataSegment segment = dataflow.getSegment(seg);
- return segment.getIndexPlan().getAllLayouts().stream().anyMatch(SecondStorageUtil::isBaseTableIndex);
- }).collect(Collectors.toList());
- if (failedSegs.isEmpty()) {
- return;
- }
- JobSubmissionException jobSubmissionException = new JobSubmissionException(
- MsgPicker.getMsg().getAddJobCheckFail());
- for (String failedSeg : failedSegs) {
- jobSubmissionException.addJobFailInfo(failedSeg,
- new KylinException(SECOND_STORAGE_ADD_JOB_FAILED, MsgPicker.getMsg().getAddExportJobFail()));
- }
- noBaseIndexSegs.forEach(seg -> jobSubmissionException.addJobFailInfo(seg,
- new KylinException(FAILED_CREATE_JOB_EXPORT_TO_TIERED_STORAGE_WITHOUT_BASE_INDEX,
- String.format(Locale.ROOT, MsgPicker.getMsg().getAddJobCheckFailWithoutBaseIndex(), seg))));
- throw jobSubmissionException;
- }
-}
diff --git a/outdated/second-storage/core/src/main/scala/io/kyligence/kap/secondstorage/ColumnMapping.scala b/outdated/second-storage/core/src/main/scala/io/kyligence/kap/secondstorage/ColumnMapping.scala
deleted file mode 100644
index 02f199e3bf..0000000000
--- a/outdated/second-storage/core/src/main/scala/io/kyligence/kap/secondstorage/ColumnMapping.scala
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.kyligence.kap.secondstorage
-
-object ColumnMapping {
- def kapColumnToSecondStorageColumn(col: String): String = "c" + col
- def secondStorageColumnToKapColumn(col: String): String = col.substring(1);
-}
diff --git a/outdated/second-storage/core/src/main/scala/io/kyligence/kap/secondstorage/SecondStorage.scala b/outdated/second-storage/core/src/main/scala/io/kyligence/kap/secondstorage/SecondStorage.scala
deleted file mode 100644
index 6c7fb0b96e..0000000000
--- a/outdated/second-storage/core/src/main/scala/io/kyligence/kap/secondstorage/SecondStorage.scala
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.kyligence.kap.secondstorage
-
-import io.kyligence.kap.secondstorage.enums.LockTypeEnum
-import io.kyligence.kap.secondstorage.metadata._
-import org.apache.kylin.common.{ForceToTieredStorage, KylinConfig, QueryContext}
-import org.apache.kylin.engine.spark.utils.JavaOptionals._
-import org.apache.kylin.engine.spark.utils.LogEx
-import org.apache.kylin.metadata.cube.model.{LayoutEntity, NDataflow}
-import org.apache.spark.sql.execution.datasources.jdbc.ShardOptions
-import org.apache.spark.sql.{DataFrame, SparkSession}
-
-import java.sql.SQLException
-import scala.collection.JavaConverters._
-import scala.util.control.NonFatal
-
-// CALL FROM CORE
-
-object SecondStorage extends LogEx {
-
- private[secondstorage] def load(pluginClassName: String): SecondStoragePlugin = {
- try {
- // scalastyle:off classforname
- val pluginClass =
- Option.apply(Class.forName(pluginClassName, true, Thread.currentThread.getContextClassLoader))
- // scalastyle:on classforname
-
- pluginClass
- .flatMap { p =>
- if (!classOf[SecondStoragePlugin].isAssignableFrom(p)) {
- logWarning(s"SecondStoragePlugin plugin class not found: $pluginClassName is not defined")
- Option(null)
- } else {
- Some(p.getDeclaredConstructor().newInstance().asInstanceOf[SecondStoragePlugin])
- }
- }
- .orNull
- } catch {
- case e: ClassNotFoundException =>
- logWarning(s"ClassNotFoundException", e)
- null
- }
- }
-
- private var secondStoragePlugin: SecondStoragePlugin = _
-
- lazy val configLoader: SecondStorageConfigLoader = {
- if (secondStoragePlugin == null) {
- throw new RuntimeException("second storage plugin is null")
- }
- secondStoragePlugin.getConfigLoader
- }
-
- def init(force: Boolean): Unit = {
- if (force || secondStoragePlugin == null) {
- secondStoragePlugin = Option(KylinConfig.getInstanceFromEnv.getSecondStorage).map(load).orNull
- }
- }
-
- def enabled: Boolean = secondStoragePlugin != null && secondStoragePlugin.ready()
-
- def tableFlowManager(config: KylinConfig, project: String): Manager[TableFlow] =
- secondStoragePlugin.tableFlowManager(config, project)
-
- def tableFlowManager(dataflow: NDataflow): Manager[TableFlow] =
- tableFlowManager(dataflow.getConfig, dataflow.getProject)
-
- def tablePlanManager(config: KylinConfig, project: String): Manager[TablePlan] =
- secondStoragePlugin.tablePlanManager(config, project)
-
- def nodeGroupManager(config: KylinConfig, project: String): Manager[NodeGroup] =
- secondStoragePlugin.nodeGroupManager(config, project)
-
- private def queryCatalog() = Option.apply(secondStoragePlugin.queryCatalog())
-
- private def forcedToTieredStorageHandler() = {
- var forcedToTieredStorage : ForceToTieredStorage = QueryContext.current.getForcedToTieredStorage
- if (forcedToTieredStorage == null) {
- forcedToTieredStorage = ForceToTieredStorage.CH_FAIL_TO_DFS
- }
- if (forcedToTieredStorage != ForceToTieredStorage.CH_FAIL_TO_DFS) {
- throw new SQLException(QueryContext.ROUTE_USE_FORCEDTOTIEREDSTORAGE)
- }
- }
- def trySecondStorage(
- sparkSession: SparkSession,
- dataflow: NDataflow,
- layout: LayoutEntity,
- pruningInfo: String): Option[DataFrame] = {
- if (!SecondStorageUtil.isModelEnable(dataflow.getProject, dataflow.getModel.getId)) {
- return None
- }
-
- if (LockTypeEnum.locked(LockTypeEnum.QUERY.name(), SecondStorageUtil.getProjectLocks(dataflow.getProject))) {
- log.info("project={} has 'QUERY' lock, can not hit clickhouse query.", dataflow.getProject)
- return Option.empty
- }
-
- var result = Option.empty[DataFrame]
- // Only support table index
- val enableSSForThisQuery = enabled && layout.getIndex.isTableIndex
- val allSegIds = pruningInfo.split(",").map(s => s.split(":")(0)).toSet.asJava
- while (enableSSForThisQuery && result.isEmpty && QueryContext.current().isRetrySecondStorage) {
- val tableData = Option.apply(enableSSForThisQuery)
- .filter(_ == true)
- .flatMap(_ =>
- tableFlowManager(dataflow)
- .get(dataflow.getUuid)
- .flatMap(f => f.getEntity(layout))
- .toOption)
- .filter { tableData =>
- tableData.containSegments(allSegIds)
- }
- if (tableData.isEmpty) {
- QueryContext.current().setRetrySecondStorage(false)
- logInfo("No table data found.")
- }
- result = tableData.flatMap(tableData =>
- tryCreateDataFrame(Some(tableData), sparkSession, dataflow, allSegIds)
- )
- }
- if (result.isDefined) {
- QueryContext.current().setLastFailed(false)
- QueryContext.current().getSecondStorageUsageMap.put(layout.getId, true)
- }
- if (enableSSForThisQuery && result.isEmpty) {
- forcedToTieredStorageHandler()
- }
-
- result
- }
-
- private def tryCreateDataFrame(tableData: Option[TableData], sparkSession: SparkSession,
- dataflow: NDataflow, allSegIds: java.util.Set[String]) = {
- try {
- var project = dataflow.getProject
- for {
- shardJDBCURLs <- tableData.map(_.getShardJDBCURLs(project, allSegIds))
- database <- tableData.map(_.getDatabase)
- table <- tableData.map(_.getTable)
- catalog <- queryCatalog()
- numPartition <- tableData.map(_.getSchemaURLSize)
- } yield {
- sparkSession.read
- .option(ShardOptions.SHARD_URLS, shardJDBCURLs)
- .option(ShardOptions.PUSHDOWN_AGGREGATE, true)
- .option(ShardOptions.PUSHDOWN_LIMIT, true)
- .option(ShardOptions.PUSHDOWN_OFFSET, true)
- .option(ShardOptions.PUSHDOWN_NUM_PARTITIONS, numPartition)
- .table(s"$catalog.$database.$table")
- }
- } catch {
- case NonFatal(e) =>
- logDebug("Failed to use second storage table-index", e)
- forcedToTieredStorageHandler()
- QueryContext.current().setLastFailed(true);
- None
- }
- }
-}
diff --git a/outdated/second-storage/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/v2/SecondStorageCatalog.scala b/outdated/second-storage/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/v2/SecondStorageCatalog.scala
deleted file mode 100644
index c0bdd75a66..0000000000
--- a/outdated/second-storage/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/v2/SecondStorageCatalog.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.execution.datasources.jdbc.v2
-
-import org.apache.kylin.guava30.shaded.common.cache.{CacheBuilder, CacheLoader, LoadingCache}
-import io.kyligence.kap.secondstorage.ColumnMapping.secondStorageColumnToKapColumn
-import io.kyligence.kap.secondstorage.NameUtil
-import io.kyligence.kap.secondstorage.SecondStorage.tableFlowManager
-import org.apache.kylin.common.KylinConfig
-import org.apache.kylin.engine.spark.utils.JavaOptionals.toRichOptional
-import org.apache.kylin.metadata.cube.model.NDataflowManager
-import org.apache.kylin.metadata.datatype.DataType
-import org.apache.spark.sql.connector.catalog.Identifier
-import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JDBCRDD}
-import org.apache.spark.sql.execution.datasources.v2.jdbc.ShardJDBCTableCatalog
-import org.apache.spark.sql.types.{BooleanType, StructType}
-
-class SecondStorageCatalog extends ShardJDBCTableCatalog {
-
- protected val cache: LoadingCache[Identifier, StructType] = CacheBuilder.newBuilder()
- .maximumSize(100L)
- .build(
- new CacheLoader[Identifier, StructType]() {
- override def load(ident: Identifier): StructType = {
- val config = KylinConfig.getInstanceFromEnv
- val project = NameUtil.recoverProject(ident.namespace()(0), config)
- val pair = NameUtil.recoverLayout(ident.name())
- val model_id = pair.getFirst
- val layout = pair.getSecond
- val dfMgr = NDataflowManager.getInstance(config, project)
- val orderedDims = dfMgr.getDataflow(model_id).getIndexPlan.getLayoutEntity(layout).getOrderedDimensions
-
- val schemaURL = tableFlowManager(config, project)
- .get(model_id)
- .flatMap(f => f.getEntity(layout))
- .toOption
- .map(_.getSchemaURL)
- .get
-
- val immutable = Map(
- JDBCOptions.JDBC_TABLE_NAME -> getTableName(ident),
- JDBCOptions.JDBC_URL -> schemaURL)
- val jdbcSchema = JDBCRDD.resolveTable(new JDBCOptions(immutable))
-
- // !!This is patch!!
- StructType(jdbcSchema.map(e => {
- orderedDims.get(secondStorageColumnToKapColumn(e.name).toInt).getType.getName match {
- case DataType.BOOLEAN => e.copy(dataType = BooleanType)
- case _ => e
- }
- })
- )
- }
- })
-
- override protected def resolveTable(ident: Identifier): Option[StructType] = {
- Option.apply(cache.get(ident))
- }
-}
diff --git a/pom.xml b/pom.xml
index 3e15a3c6c7..7563f383b2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -385,12 +385,6 @@
<!-- Apache POI -->
<poi.version>4.1.2</poi.version>
- <!-- clickhouse -->
- <clickhouse-native-jdbc.version>2.6.5</clickhouse-native-jdbc.version>
- <clickhouse-jdbc.version>0.2.4</clickhouse-jdbc.version>
- <clickhouse.group>ru.yandex.clickhouse</clickhouse.group>
- <clickhouse.artifact>clickhouse-jdbc</clickhouse.artifact>
-
<external.version>5.0.0</external.version>
<iam-user-manager.version>2.1.0</iam-user-manager.version>
@@ -444,7 +438,6 @@
<module>src/tool</module>
<module>src/datasource-sdk</module>
<module>src/streaming-sdk</module>
- <module>outdated/second-storage/core</module>
<module>src/modeling-service</module>
<module>src/query-service</module>
<module>src/datasource-service</module>
@@ -550,27 +543,6 @@
<artifactId>kylin-metadata-feign-api</artifactId>
<version>${project.version}</version>
</dependency>
- <!-- second storage -->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kap-second-storage-core</artifactId>
-<!-- <version>5.0.0-beta</version>-->
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-job</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-spark-common</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-common-service</artifactId>
@@ -723,37 +695,6 @@
<version>${mssql-jdbc.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.github.housepower</groupId>
- <artifactId>clickhouse-native-jdbc</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.lz4</groupId>
- <artifactId>lz4-java</artifactId>
- </exclusion>
- </exclusions>
- <version>${clickhouse-native-jdbc.version}</version>
- </dependency>
- <dependency>
- <groupId>ru.yandex.clickhouse</groupId>
- <artifactId>clickhouse-jdbc</artifactId>
- <exclusions>
- <exclusion>
- <artifactId>jackson-databind</artifactId>
- <groupId>com.fasterxml.jackson.core</groupId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>net.jpountz.lz4</groupId>
- <artifactId>lz4</artifactId>
- </exclusion>
- </exclusions>
- <version>${clickhouse-jdbc.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
@@ -864,12 +805,6 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kap-second-storage-clickhouse</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-metadata</artifactId>
@@ -1013,10 +948,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec-haproxy</artifactId>
- </exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>*</artifactId>
@@ -2776,10 +2707,6 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
<exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec-haproxy</artifactId>
- </exclusion>
<exclusion>
<artifactId>jetty-plus</artifactId>
<groupId>org.eclipse.jetty</groupId>
@@ -2857,10 +2784,6 @@
<type>test-jar</type>
<scope>test</scope>
<exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec-haproxy</artifactId>
- </exclusion>
<exclusion>
<artifactId>jetty-plus</artifactId>
<groupId>org.eclipse.jetty</groupId>
@@ -3082,56 +3005,6 @@
<version>${transmittable-thread-local.version}</version>
</dependency>
- <!-- Introduced for clickhouse test -->
- <dependency>
- <groupId>org.eclipse.jetty.toolchain</groupId>
- <artifactId>jetty-test-helper</artifactId>
- <version>${jetty.toolchain.test-helper.version}</version>
- <exclusions>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- </exclusions>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.testcontainers</groupId>
- <artifactId>junit-jupiter</artifactId>
- <version>${test.container.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- </exclusion>
- </exclusions>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.testcontainers</groupId>
- <artifactId>jdbc</artifactId>
- <version>${test.container.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.testcontainers</groupId>
- <artifactId>clickhouse</artifactId>
- <version>${test.container.version}</version>
- <exclusions>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- </exclusions>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.testcontainers</groupId>
- <artifactId>nginx</artifactId>
- <version>${test.container.version}</version>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
diff --git a/src/common-booter/src/main/resources/config/config_library.csv b/src/common-booter/src/main/resources/config/config_library.csv
index 5fa34f3965..59637b2d33 100644
--- a/src/common-booter/src/main/resources/config/config_library.csv
+++ b/src/common-booter/src/main/resources/config/config_library.csv
@@ -397,12 +397,6 @@ kylin.restclient.connection.default-max-per-route,Int,http请求到达指定机
kylin.restclient.connection.max-total,Int,http请求连接池最大连接数,HTTP request connection pool maximum number of connections,TRUE,TRUE
kylin.scheduler.schedule-job-timeout-minute,Int,作业超时时长，默认没有超时,"The job is timeout, and there is no timeout for default",TRUE,TRUE
kylin.scheduler.schedule-limit-per-minute,Int,限制调度器事件处理的频率，默认每分钟10次,"Limit the frequency of processing the scheduling event, 10 times per minute by default",TRUE,TRUE
-kylin.second-storage.diag-log-matcher,String,分层存储诊断包的日志文件匹配规则,Log file matching rules for layered storage diagnostic packages,TRUE,TRUE
-kylin.second-storage.diag-max-compressed-file,Int,分层存储诊断包允许的最大压缩文件个数,The number of maximum compression files allowed by layered storage diagnostic packages,TRUE,TRUE
-kylin.second-storage.load-deduplication-window,Int,分层存储的重复数据删除窗口,Repeated data delete window for layered storage,TRUE,TRUE
-kylin.second-storage.load-retry,Int,重试次数,number of retries,TRUE,TRUE
-kylin.second-storage.load-retry-interval,Int,重试时间,Retry time,TRUE,TRUE
-kylin.second-storage.ssh-identity-path,String,分层存储的私钥路径,Private key path for layered storage,TRUE,TRUE
kylin.security.acl.admin-role,String,将一个 LDAP 群组映射成管理员角色（大小写敏感）,Imam into a LDAP group into an administrator role (sensitivity),TRUE,TRUE
kylin.security.allow-non-admin-generate-query-diag-package,boolean,是否允许非管理员打查询诊断包,Whether the non -administrator is allowed to query the diagnostic package,TRUE,TRUE
kylin.security.allow-project-admin-grant-acl,boolean,是否允许每个项目中的项目管理员给用户或用户组分配表/行/列访问权限,Whether the project administrator in each project is allowed to distribute the table/row/line access permission to the user or user group,TRUE,TRUE
@@ -545,7 +539,6 @@ server.port,Int,服务启动的端口,The port of the service start,TRUE,TRUE
spring.session.store-type,String,spring session 存储类型,Spring session storage type,TRUE,TRUE
kylin.engine.dynamic-resource-plan-enabled,boolean,在FI平台使用动态资源计划时候，有可能导致无法分配资源，从而构建失败,"When using a dynamic resource plan on the FI platform, it may lead to the failure to allocate resources and build a failure",TRUE,TRUE
kylin.engine.spark-conf.spark.sql.adaptive.coalescePartitions.minPartitionNum,Int,此参数默认值为 Application 的 Default Parallelism，计算公式为 instance * core，设置此参数为较低值（建议为1）时，可修正由于默认值过大而造成的最终分区数量过大（产生过多tasks），进而影响到整体构建性能。,"The default value of this parameter is the Default Paullelism of Application. The calculation formula is Instance * Core. When setting this parameter to a lower value (Suggestion is 1), the number of final partitions caused by excessive default value can Tasks), which affects the [...]
-kylin.second-storage.node-replica,Int,分层存储节点副本数量,Number of copies of storage nodes,TRUE,TRUE
kylin.cube.merge-segment-storage-threshold,Double,"segment合并阈值检查百分比，取值范围为[0,1]，为0时代表不执行检查逻辑","Segment merged threshold inspection percentage, the range of the value is [0,1], which is not performed by the 0 era.",TRUE,TRUE
kylin.query.engine.clean-spark-ui-zombie-job-enabled,boolean,清理查询 spark 应用的僵尸job,Clean up the zombie job of spark application,TRUE,TRUE
kylin.job.use-bigint-as-timestamp-for-partition-column,boolean,支持timestamp格式的bigint作为时间分区列，之前行为支持，改动后默认不支持，可以设置为true回滚为之前行为,"BIGINT supports the TimeStamp format as the time partition column. The previous behavior supports, the default is not supported after the change, it can be set to roll back as the previous behavior",TRUE,TRUE
diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java
index fb6882b74c..cc857a15dc 100644
--- a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java
+++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NAdminController.java
@@ -78,12 +78,6 @@ public class NAdminController extends NBasicController {
propertyKeys.add("kylin.table.load-threshold-enabled");
propertyKeys.add("kylin.index.enable-operator-design");
- // add second storage
- if (StringUtils.isNotEmpty(KylinConfig.getInstanceFromEnv().getSecondStorage())) {
- propertyKeys.add("kylin.second-storage.class");
- propertyKeys.add("kylin.second-storage.query-metric-collect");
- }
-
if (!KylinConfig.getInstanceFromEnv().isAllowedNonAdminGenerateQueryDiagPackage()) {
propertyKeys.add("kylin.security.allow-non-admin-generate-query-diag-package");
}
diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
index 643fd3a506..19628ffa44 100644
--- a/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
+++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
@@ -27,7 +27,6 @@ import static org.apache.kylin.common.exception.ServerErrorCode.FAILED_DOWNLOAD_
import static org.apache.kylin.common.exception.ServerErrorCode.INCORRECT_PROJECT_MODE;
import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_DATA_BINDER_AUTO_GROW_COLLECTION_LIMIT;
import static org.apache.kylin.common.exception.ServerErrorCode.INVALID_PARAMETER;
-import static org.apache.kylin.common.exception.ServerErrorCode.LOW_LEVEL_LICENSE;
import static org.apache.kylin.common.exception.ServerErrorCode.UNSUPPORTED_STREAMING_OPERATION;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.ARGS_TYPE_CHECK;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.DATETIME_FORMAT_EMPTY;
@@ -83,7 +82,6 @@ import org.apache.kylin.common.exception.FeignErrorResponse;
import org.apache.kylin.common.exception.FeignRpcException;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.ServerErrorCode;
-import org.apache.kylin.common.extension.KylinInfoExtension;
import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.transaction.TransactionException;
@@ -710,16 +708,6 @@ public class NBasicController {
return host;
}
- public void checkKylinInfo(boolean enableSecondStorage) {
- if (enableSecondStorage) {
- boolean checkKylinInfo = KylinInfoExtension.getFactory().checkKylinInfo();
- if (!checkKylinInfo) {
- Message msg = MsgPicker.getMsg();
- throw new KylinException(LOW_LEVEL_LICENSE, msg.getLowLevelLicenseMessage());
- }
- }
- }
-
@InitBinder
private void initBinder(WebDataBinder binder) {
int autoGrowCollectionLimit = KylinConfig.getInstanceFromEnv().getDataBinderAutoGrowCollectionLimit();
diff --git a/src/common-server/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java b/src/common-server/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java
index b90eb3360c..7436f75073 100644
--- a/src/common-server/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java
+++ b/src/common-server/src/test/java/org/apache/kylin/rest/controller/NBasicControllerTest.java
@@ -19,7 +19,6 @@
package org.apache.kylin.rest.controller;
import static org.apache.kylin.common.exception.CommonErrorCode.UNKNOWN_ERROR_CODE;
-import static org.apache.kylin.common.exception.ServerErrorCode.LOW_LEVEL_LICENSE;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.DATETIME_FORMAT_EMPTY;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.DATETIME_FORMAT_PARSE_ERROR;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.INTEGER_NON_NEGATIVE_CHECK;
@@ -31,7 +30,6 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.TIME_INVALI
import static org.apache.kylin.common.exception.code.ErrorCodeServer.USER_AUTH_INFO_NOTFOUND;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -43,7 +41,6 @@ import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.extension.KylinInfoExtension;
-import org.apache.kylin.common.msg.Message;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
@@ -63,7 +60,6 @@ import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mockito;
-import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -73,8 +69,6 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.result.MockMvcResultMatchers;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
-import lombok.val;
-
@RunWith(PowerMockRunner.class)
@PowerMockIgnore({ "com.sun.security.*", "org.w3c.*", "javax.xml.*", "org.xml.*", "org.apache.cxf.*",
"javax.management.*", "javax.script.*", "org.apache.hadoop.*", "javax.security.*", "java.security.*",
@@ -339,31 +333,4 @@ public class NBasicControllerTest extends NLocalFileMetadataTestCase {
Assert.assertEquals("ip", nBasicController.decodeHost("ip"));
}
- @Test
- public void testCheckLicenseLevel() throws Exception {
- nBasicController.checkKylinInfo(false);
-
- PowerMockito.mockStatic(KylinInfoExtension.class);
- val factory = Mockito.mock(KylinInfoExtension.Factory.class);
- Mockito.when(factory.checkKylinInfo()).thenReturn(true);
- PowerMockito.when(KylinInfoExtension.getFactory()).thenReturn(factory);
- nBasicController.checkKylinInfo(true);
-
- checkErrorMessage(factory, "en", Message.getInstance());
- // checkErrorMessage(factory, "cn", CnMessage.getInstance());
- }
-
- private void checkErrorMessage(KylinInfoExtension.Factory factory, String lang, Message message) {
- try {
- MsgPicker.setMsg(lang);
- Mockito.when(factory.checkKylinInfo()).thenReturn(false);
- nBasicController.checkKylinInfo(true);
- fail();
- } catch (Exception e) {
- Assert.assertTrue(e instanceof KylinException);
- val kylinException = (KylinException) e;
- Assert.assertEquals(LOW_LEVEL_LICENSE.toErrorCode(), kylinException.getErrorCode());
- Assert.assertEquals(message.getLowLevelLicenseMessage(), kylinException.getMessage());
- }
- }
}
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryMetricsListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryMetricsListener.java
index e544eab037..a863c9b681 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryMetricsListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/initialize/QueryMetricsListener.java
@@ -88,8 +88,7 @@ public class QueryMetricsListener {
MetricsTag.SUCCEED.getVal(), queryMetric.isSucceed() + "", MetricsTag.HIT_SNAPSHOT.getVal(),
queryMetric.isTableSnapshotUsed() + "", MetricsTag.PROJECT.getVal(),
queryMetric.getProjectName(), MetricsTag.CONSTANTS.getVal(),
- MetricsTag.CONSTANTS.getVal().equalsIgnoreCase(queryMetric.getEngineType()) + "",
- MetricsTag.HIT_SECOND_STORAGE.getVal(), queryMetric.isSecondStorage() + "")
+ MetricsTag.CONSTANTS.getVal().equalsIgnoreCase(queryMetric.getEngineType()) + "")
.distributionStatisticExpiry(Duration.ofDays(1))
.sla(KylinConfig.getInstanceFromEnv().getMetricsQuerySlaSeconds()).description("Query duration")
.register(meterRegistry).record(queryMetric.getQueryDuration() * 1.0 / 1000);
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/request/SQLRequest.java b/src/common-service/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index e4fae4b462..93b03d3413 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -68,7 +68,6 @@ public class SQLRequest implements Serializable, ProjectInsensitiveRequest, Vali
@JsonProperty("file_name")
private String fileName = "result";
- private Integer forcedToTieredStorage; //0:CH->DFS; 1:CH->pushDown; 2:CH->return error
@JsonProperty("include_header")
private boolean includeHeader;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/response/ProjectConfigResponse.java b/src/common-service/src/main/java/org/apache/kylin/rest/response/ProjectConfigResponse.java
index 8ff68811af..d099305211 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/response/ProjectConfigResponse.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/response/ProjectConfigResponse.java
@@ -28,7 +28,6 @@ import org.apache.kylin.metadata.model.VolatileRange;
import com.fasterxml.jackson.annotation.JsonProperty;
-import io.kyligence.kap.secondstorage.response.SecondStorageNode;
import lombok.Getter;
import lombok.Setter;
@@ -69,10 +68,6 @@ public class ProjectConfigResponse {
@JsonProperty("snapshot_automatic_refresh_trigger_second")
private String snapshotAutoRefreshTriggerSecond = "0";
- @JsonProperty("second_storage_enabled")
- private Boolean secondStorageEnabled;
- @JsonProperty("second_storage_nodes")
- private List<SecondStorageNode> secondStorageNodes;
@JsonProperty("runner_class_name")
private String runnerClassName;
@JsonProperty("converter_class_names")
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index 151fc595bd..c690e21118 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -152,7 +152,6 @@ import org.springframework.web.multipart.MultipartFile;
import com.fasterxml.jackson.core.JsonProcessingException;
-import io.kyligence.kap.secondstorage.SecondStorageUtil;
import lombok.SneakyThrows;
import lombok.val;
@@ -657,11 +656,6 @@ public class ProjectService extends BasicService {
response.setPackageVersion(KylinVersion.getCurrentVersion().toString());
response.setPackageTimestamp(infos.getSecond());
- if (SecondStorageUtil.isGlobalEnable()) {
- response.setSecondStorageEnabled(SecondStorageUtil.isProjectEnable(project));
- response.setSecondStorageNodes(SecondStorageUtil.listProjectNodes(project));
- }
-
return response;
}
@@ -945,11 +939,6 @@ public class ProjectService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
@Transaction(project = 0)
public void dropProject(String project, HttpHeaders headers) {
- if (SecondStorageUtil.isProjectEnable(project)) {
- throw new KylinException(PROJECT_DROP_FAILED,
- String.format(Locale.ROOT, MsgPicker.getMsg().getProjectDropFailedSecondStorageEnabled(), project));
- }
-
ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
List<String> jobIds = executableManager
.getExecutablePOsByStatus(Lists.newArrayList(ExecutableState.RUNNING, ExecutableState.PENDING,
@@ -977,6 +966,8 @@ public class ProjectService extends BasicService {
private void deleteProjectRelatedMeta(String project) {
// delete query history id offset
QueryHistoryIdOffsetManager.getInstance(project).delete();
+ // delete favorite rule
+ FavoriteRuleManager.getInstance(project).deleteByProject();
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#project, 'ADMINISTRATION')")
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
index 5af2cfdc4d..77f0421ed8 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
@@ -31,11 +31,9 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.mockito.Mockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.ImportResource;
@@ -50,17 +48,15 @@ import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.web.WebAppConfiguration;
-import io.kyligence.kap.secondstorage.SecondStorageUpdater;
-
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes = ServiceTestBase.SpringConfig.class)
@WebAppConfiguration(value = "../common-service/src/test/resources")
@TestPropertySource(properties = {"spring.cloud.nacos.discovery.enabled = false"})
@TestPropertySource(properties = {"spring.session.store-type = NONE"})
-@ActiveProfiles({ "testing", "test" })
-@PowerMockIgnore({ "com.sun.security.*", "org.w3c.*", "javax.xml.*", "org.xml.*", "org.apache.*", "org.w3c.dom.*",
+@ActiveProfiles({"testing", "test"})
+@PowerMockIgnore({"com.sun.security.*", "org.w3c.*", "javax.xml.*", "org.xml.*", "org.apache.*", "org.w3c.dom.*",
"org.apache.cxf.*", "javax.management.*", "javax.script.*", "org.apache.hadoop.*", "javax.security.*",
- "java.security.*", "javax.crypto.*", "javax.net.ssl.*", "org.apache.kylin.common.asyncprofiler.AsyncProfiler" })
+ "java.security.*", "javax.crypto.*", "javax.net.ssl.*", "org.apache.kylin.common.asyncprofiler.AsyncProfiler"})
public class ServiceTestBase extends NLocalFileMetadataTestCase {
@Autowired
@@ -120,12 +116,8 @@ public class ServiceTestBase extends NLocalFileMetadataTestCase {
@Configuration
@ComponentScan("org.apache.kylin.rest")
- @ImportResource(locations = { "classpath:applicationContext.xml", "classpath:kylinSecurity.xml" })
+ @ImportResource(locations = {"classpath:applicationContext.xml", "classpath:kylinSecurity.xml"})
@EnableAsync
public static class SpringConfig {
- @Bean
- public SecondStorageUpdater getSecondStorageUpdater() {
- return Mockito.mock(SecondStorageUpdater.class, invocationOnMock -> "");
- }
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/ForceToTieredStorage.java b/src/core-common/src/main/java/org/apache/kylin/common/ForceToTieredStorage.java
deleted file mode 100644
index 2afc62bcf8..0000000000
--- a/src/core-common/src/main/java/org/apache/kylin/common/ForceToTieredStorage.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.common;
-
-public enum ForceToTieredStorage {
- CH_FAIL_TO_DFS,
- CH_FAIL_TO_PUSH_DOWN,
- CH_FAIL_TO_RETURN,
- CH_FAIL_TAIL,
-}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 003bfce8b9..b7bb41c173 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -545,55 +545,6 @@ public abstract class KylinConfigBase implements Serializable {
public boolean isMetadataCompressEnabled() {
return Boolean.parseBoolean(getOptional("kylin.metadata.compress.enabled", TRUE));
}
-
- public String getSecondStorage() {
- return getOptional("kylin.second-storage.class", null);
- }
-
- public String getSecondStorageDiagLogMatcher() {
- return getOptional("kylin.second-storage.diag-log-matcher", "*-server.log");
- }
-
- public int getSecondStorageDiagMaxCompressedFile() {
- return Integer.parseInt(getOptional("kylin.second-storage.diag-max-compressed-file", "2"));
- }
-
- public String getSecondStorageSshIdentityPath() {
- return getOptional("kylin.second-storage.ssh-identity-path", "~/.ssh/id_rsa");
- }
-
- public int getSecondStorageLoadDeduplicationWindow() {
- return Integer.parseInt(getOptional("kylin.second-storage.load-deduplication-window", "0"));
- }
-
- public int getSecondStorageLoadRetry() {
- return Integer.parseInt(getOptional("kylin.second-storage.load-retry", "3"));
- }
-
- public int getSecondStorageLoadRetryInterval() {
- return Integer.parseInt(getOptional("kylin.second-storage.load-retry-interval", "30000"));
- }
-
- public boolean getSecondStorageQueryMetricCollect() {
- return Boolean.parseBoolean(getOptional("kylin.second-storage.query-metric-collect", TRUE));
- }
-
- public int getSecondStorageQueryPushdownLimit() {
- return Integer.parseInt(getOptional("kylin.second-storage.query-pushdown-limit", "0"));
- }
-
- public boolean getSecondStorageUseLowCardinality() {
- return Boolean.parseBoolean(getOptional("kylin.second-storage.use-low-cardinality", TRUE));
- }
-
- public long getSecondStorageLowCardinalityNumber() {
- return Long.parseLong(getOptional("kylin.second-storage.low-cardinality-number", "10000"));
- }
-
- public long getSecondStorageHighCardinalityNumber() {
- return Long.parseLong(getOptional("kylin.second-storage.high-cardinality-number", ONE_HUNDRED_THOUSAND));
- }
-
public int getMetadataCacheMaxNum() {
return Integer.parseInt(getOptional("kylin.metadata.cache.max-num", String.valueOf(Integer.MAX_VALUE)));
}
@@ -3927,17 +3878,6 @@ public abstract class KylinConfigBase implements Serializable {
public boolean isUnitOfWorkSimulationEnabled() {
return Boolean.parseBoolean(getOptional("kylin.env.unitofwork-simulation-enabled", FALSE));
}
-
- public ForceToTieredStorage getSystemForcedToTieredStorage() {
- int i = Integer.parseInt(getOptional("kylin.second-storage.route-when-ch-fail", "0"));
- return ForceToTieredStorage.values()[i];
- }
-
- public ForceToTieredStorage getProjectForcedToTieredStorage() {
- int i = Integer.parseInt(getOptional("kylin.second-storage.route-when-ch-fail"));
- return ForceToTieredStorage.values()[i];
- }
-
public long getClusterManagerHealthCheckMaxTimes() {
return Long.parseLong(getOptional("kylin.engine.cluster-manager-health-check-max-times", "10"));
}
@@ -4061,47 +4001,6 @@ public abstract class KylinConfigBase implements Serializable {
public int getJobTagMaxSize() {
return Integer.parseInt(this.getOptional("kylin.job.tag-max-size", "1024"));
}
-
- public String getJobSchedulerMode() {
- return getOptional("kylin.engine.job-scheduler-mode", "DAG");
- }
-
- public String getKylinEngineSegmentOnlineMode() {
- return getOptional("kylin.engine.segment-online-mode", SegmentOnlineMode.DFS.toString());
- }
-
- public int getSecondStorageLoadThreadsPerJob() {
- int process = Integer.parseInt(getOptional("kylin.second-storage.load-threads-per-job", "3"));
- if (process <= 0) {
- process = 1;
- }
- return process;
- }
-
- public int getSecondStorageCommitThreadsPerJob() {
- int process = Integer.parseInt(getOptional("kylin.second-storage.commit-threads-per-job", "10"));
- if (process <= 0) {
- process = 1;
- }
- return process;
- }
-
- public long getSecondStorageWaitIndexBuildSecond() {
- return Long.parseLong(getOptional("kylin.second-storage.wait-index-build-second", "10"));
- }
-
- public String getSecondStorageJDBCKeepAliveTimeout() {
- return getOptional("kylin.second-storage.jdbc-keep-alive-timeout", "600000");
- }
-
- public String getSecondStorageJDBCSocketTimeout() {
- return getOptional("kylin.second-storage.jdbc-socket-timeout", "600000");
- }
-
- public String getSecondStorageJDBCExtConfig() {
- return getOptional("kylin.second-storage.jdbc-ext-config", "connect_timeout=3");
- }
-
public long getRoutineOpsTaskTimeOut() {
return TimeUtil.timeStringAs(getOptional("kylin.metadata.ops-cron-timeout", "4h"), TimeUnit.MILLISECONDS);
}
@@ -4146,7 +4045,6 @@ public abstract class KylinConfigBase implements Serializable {
return TimeUtil.timeStringAs(getOptional("kylin.metrics.hdfs-periodic-calculation-interval", "5m"),
TimeUnit.MILLISECONDS);
}
-
public String getKubernetesUploadPath() {
return getOptional(getKubernetesUploadPathKey());
}
@@ -4162,29 +4060,6 @@ public abstract class KylinConfigBase implements Serializable {
public boolean isSkipResourceCheck() {
return Boolean.parseBoolean(getOptional("kylin.build.resource.skip-resource-check", FALSE));
}
-
- public int getSecondStorageSkippingIndexGranularity() {
- int granularity = Integer.parseInt(getOptional("kylin.second-storage.skipping-index.granularity", "3"));
- return granularity <= 0 ? 3 : granularity;
- }
-
- public String getSecondStorageSkippingIndexBloomFilter() {
- return getOptional("kylin.second-storage.skipping-index.bloom-filter", "0.025");
- }
-
- public int getSecondStorageSkippingIndexSet() {
- int size = Integer.parseInt(getOptional("kylin.second-storage.skipping-index.set", "100"));
- return size <= 0 ? 100 : size;
- }
-
- public boolean getSecondStorageIndexAllowNullableKey() {
- return Boolean.parseBoolean(getOptional("kylin.second-storage.allow-nullable-skipping-index", TRUE));
- }
-
- public int getSecondStorageWaitLockTimeout() {
- return Integer.parseInt(getOptional("kylin.second-storage.wait-lock-timeout", "180"));
- }
-
public boolean isBuildSegmentOverlapEnabled() {
return Boolean.parseBoolean(getOptional("kylin.build.segment-overlap-enabled", FALSE));
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index 4b2ac9cb50..b41d4e40df 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -19,8 +19,6 @@
package org.apache.kylin.common;
import java.io.Closeable;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -52,7 +50,6 @@ public class QueryContext implements Closeable {
public static final String PUSHDOWN_OBJECT_STORAGE = "OBJECT STORAGE";
public static final long DEFAULT_NULL_SCANNED_DATA = -1L;
- public static final String ROUTE_USE_FORCEDTOTIEREDSTORAGE = "should route use forcedToTieredStorage";
private static final TransmittableThreadLocal<QueryContext> contexts //
= new TransmittableThreadLocal<QueryContext>() {
@@ -112,38 +109,9 @@ public class QueryContext implements Closeable {
@Setter
private boolean isForModeling;
- @Getter
- @Setter
- private ForceToTieredStorage forcedToTieredStorage;
- /**
- * mark table index use second storage, key is layout id
- */
- @Getter
- @Setter
- private Map<Long, Boolean> secondStorageUsageMap = new HashMap<>();
-
@Getter
@Setter
private boolean forceTableIndex = false;
-
- // record second storage partition which used
- @Getter
- private final List<Integer> usedPartitionIndexes = new ArrayList<>();
-
- // record second storage partition which used
- @Getter
- @Setter
- private List<String> secondStorageUrls;
-
- // record last partition status
- @Getter
- @Setter
- private boolean lastFailed = false;
-
- @Getter
- @Setter
- private boolean retrySecondStorage = true;
-
@Getter
@Setter
private Map<String, Boolean> unmatchedJoinDigest = new ConcurrentHashMap<>();
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/JobErrorCode.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/JobErrorCode.java
index f337611f47..c056aafb3b 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/JobErrorCode.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/JobErrorCode.java
@@ -33,10 +33,6 @@ public enum JobErrorCode implements ErrorCodeSupplier {
PROFILING_STATUS_ERROR("KE-030001011"),
PROFILING_COLLECT_TIMEOUT("KE-030001012"),
- SECOND_STORAGE_SEGMENTS_CONFLICTS("KE-030002001"),
- SECOND_STORAGE_JOB_EXISTS("KE-030002002"),
- SECOND_STORAGE_PROJECT_JOB_EXISTS("KE-030002003"),
-
// cost based planner error code
COST_BASED_PLANNER_ERROR("KE-030003001");
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/QueryErrorCode.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/QueryErrorCode.java
index 9868b4e6b7..2a4a23b929 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/QueryErrorCode.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/QueryErrorCode.java
@@ -61,12 +61,7 @@ public enum QueryErrorCode implements ErrorCodeSupplier {
// 20080XXX query limit
REFUSE_NEW_QUERY("KE-020080001"),
- BIG_QUERY_DECIDE("KE-020008002"),
-
- // 20090XXX forcedToTieredStorage route query
- FORCED_TO_TIEREDSTORAGE_AND_FORCE_TO_INDEX("KE-020090001"),
- FORCED_TO_TIEREDSTORAGE_RETURN_ERROR("KE-020090002"),
- FORCED_TO_TIEREDSTORAGE_INVALID_PARAMETER("KE-020090003");
+ BIG_QUERY_DECIDE("KE-020008002");
private final ErrorCode errorCode;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
index a4a1b6bf61..8d08db596c 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
@@ -37,6 +37,7 @@ public enum ServerErrorCode implements ErrorCodeSupplier {
PROJECT_NAME_ILLEGAL("KE-010001005"), //
FAILED_CREATE_PROJECT("KE-010001006"), //
INCORRECT_PROJECT_MODE("KE-010001007"), //
+ PROJECT_DROP_FAILED("KE-010001008"), //
// 10002XXX model
FAILED_UPDATE_MODEL("KE-010002005"), //
@@ -257,29 +258,6 @@ public enum ServerErrorCode implements ErrorCodeSupplier {
UNSUPPORTED_STREAMING_OPERATION("KE-010035014"), //
READ_KAFKA_JAAS_FILE_ERROR("KE-010035015"), //
- // 10037XXX second storage
- SECOND_STORAGE_NODE_NOT_AVAILABLE("KE-010037001"), //
- BASE_TABLE_INDEX_NOT_AVAILABLE("KE-010037002"), //
- PARTITION_COLUMN_NOT_AVAILABLE("KE-010037003"), //
- SECOND_STORAGE_ADD_JOB_FAILED("KE-010037004"), //
- SECOND_STORAGE_PROJECT_STATUS_ERROR("KE-010037005"), //
- SECOND_STORAGE_DATA_NOT_EXIST("KE-010037006"), //
- SECOND_STORAGE_PROJECT_LOCK_FAIL("KE-010037007"), //
- SECOND_STORAGE_PROJECT_LOCKING("KE-010037008"), //
- PROJECT_DROP_FAILED("KE-010037009"), //
- JOB_RESTART_FAILED("KE-010037010"), //
- SEGMENT_DROP_FAILED("KE-010037011"), //
- JOB_RESUME_FAILED("KE-010037012"), //
- SECOND_STORAGE_DELETE_NODE_FAILED("KE-010037013"), //
- PROJECT_NOT_ENABLE("KE-010037014"), //
- SECOND_STORAGE_INDEX_NOT_SUPPORT("KE-010037015"), //
- SECOND_STORAGE_ORDER_BY_INDEX_HAS_DATA("KE-010037016"), //
- SECOND_STORAGE_LAYOUT_NOT_EXIST("KE-010037017"), //
- SECOND_STORAGE_LAYOUT_NOT_BASE_TABLE_INDEX("KE-010037018"), //
- SECOND_STORAGE_INDEX_NOT_ALLOW_NULLABLE("KE-010037019"), //
- JOB_PAUSE_FAILED("KE-010037020"), //
- SECOND_STORAGE_NOT_SUPPORT_TYPE("KE-010037021"), //
-
//10038XXX system profile
SYSTEM_PROFILE_ABNORMAL_DATA("KE-010038001"), //
SPARK_FAILURE("KE-010038002"), //
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
index f4b1dff0fc..446fe27a8d 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/exception/code/ErrorCodeServer.java
@@ -40,8 +40,6 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
MODEL_SUM_LC_INVALID_DATA_TYPE("KE-010002303"),
MODEL_SUM_LC_INVALID_TIMESTAMP_TYPE("KE-010002304"),
MODEL_NAME_TOO_LONG("KE-010002305"),
- MODEL_SECOND_STORAGE_PARTITION_INVALID("KE-010002306"),
- PARTITION_SECOND_STORAGE_PARTITION_INVALID("KE-010002307"),
SCD2_MODEL_CAN_ONLY_CONNECT_BY_AND("KE-010002401"),
SCD2_MODEL_CONTAINS_ILLEGAL_EXPRESSIONS("KE-010002402"),
SCD2_CONDITION_MUST_APPEAR_IN_PAIRS("KE-010002403"),
@@ -77,8 +75,6 @@ public enum ErrorCodeServer implements ErrorCodeProducer {
SEGMENT_INDEX_CONFLICT_PARAMETER("KE-010022220"),
SEGMENT_INDEX_STATUS_INVALID("KE-010022221"),
SEGMENT_SINGLE_JOB_THRESHOLD("KE-010022222"),
- SEGMENT_SECOND_STORAGE_PARTITION_INVALID("KE-010022223"),
- SEGMENT_STATUS_ILLEGAL("KE-010022224"),
// 100072XX table
TABLE_RELOAD_MODEL_RETRY("KE-010007204"),
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
index 32b552bd0a..98736f31f6 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
@@ -279,11 +279,6 @@ public class CnMessage extends Message {
return "项目名 \"%s\" 已存在。请重新命名。";
}
- @Override
- public String getProjectDropFailedSecondStorageEnabled() {
- return "无法删除项目 \"%s\"，请先关闭分层存储。";
- }
-
@Override
public String getProjectDropFailedJobsNotKilled() {
return "无法删除项目 \"%s\"，请终止相关任务后再试。";
@@ -547,7 +542,7 @@ public class CnMessage extends Message {
@Override
public String getLowLevelLicenseMessage() {
- return "无法使用分层存储，如需使用请先升级为 Kyligence 高级版";
+ return "如需使用该功能请先升级为 Kyligence 高级版";
}
@Override
@@ -1273,82 +1268,11 @@ public class CnMessage extends Message {
public String getQueryHistoryColumnMeta() {
return "查询开始时间,查询耗时,查询 ID,SQL 语句,查询对象,查询状态,查询节点,查询用户,查询信息\n";
}
-
- @Override
- public String getSecondStorageJobExists() {
- return "当前无法关闭分层存储。模型 “%s” 存在正在运行的任务，请检查后再试。\n";
- }
-
- @Override
- public String getSecondStorageConcurrentOperate() {
- return "存在相关的分层存储的任务正在运行，请稍后重试。";
- }
-
- @Override
- public String getSecondStorageProjectJobExists() {
- return "当前无法关闭分层存储。项目 “%s” 存在正在运行的任务，请检查后再试。\n";
- }
-
- @Override
- public String getSecondStorageProjectEnabled() {
- return "项目 %s 未开启分层存储。";
- }
-
- @Override
- public String getSecondStorageModelEnabled() {
- return "模型 %s 未开启分层存储。";
- }
-
- @Override
- public String getSecondStorageLayoutNotExist() {
- return "不存在 Layout id %s。";
- }
-
- @Override
- public String getSecondStorageLayoutNotBaseTableIndex() {
- return "Layout id %s 不是和分层存储一致的基础明细索引。";
- }
-
- @Override
- public String getSecondStorageNotSupportType(String dataType) {
- return String.format(Locale.ROOT, "Skipping Index不支持'%s'列类型。", dataType);
- }
-
- @Override
- public String getSecondStorageSegmentWithoutBaseIndex() {
- return "Segment 中缺少基础明细索引，请添加后重试。";
- }
-
- @Override
- public String getSecondStorageDeleteNodeFailed() {
- return "节点%s存在数据，大小为%d bytes";
- }
-
- @Override
- public String getInvalidLowCardinalityDataType() {
- return "无效的数据类型，当前仅支持 LowCardinality(Nullable(String)) 或 Nullable(String) 的数据类型。";
- }
-
- @Override
- public String getJobRestartFailed() {
- return "分层存储任务不支持重启操作。\n";
- }
-
- @Override
- public String getJobResumeFailed() {
- return "分层存储任务暂时不能恢复，请稍后再试。\n";
- }
-
@Override
public String getJobPauseFailed() {
return "该类型任务不支持暂停操作";
}
- @Override
- public String getSegmentDropFailed() {
- return "Segment 正在导入分层存储中。请稍后重试。\n";
- }
-
@Override
public String getInvalidBrokerDefinition() {
return "Broker 信息不可为空，请检查后重试。";
@@ -1413,22 +1337,6 @@ public class CnMessage extends Message {
public String getStreamingIndexesConvert() {
return "流数据模型暂无法转换为优化建议。";
}
-
- @Override
- public String getForcedToTieredstorageAndForceToIndex() {
- return "“force_to_index=ture“ 时，查询使用分层存储失败时不能下压，“forcedToTieredStorage“=1 或者 conf=1 无效，请修改后重试";
- }
-
- @Override
- public String getForcedToTieredstorageReturnError() {
- return "查询失败。分层存储不可用，请修复后重试查询";
- }
-
- @Override
- public String getForcedToTieredstorageInvalidParameter() {
- return "无效的参数值，请修改后重试";
- }
-
@Override
public String getParameterEmpty() {
return PARAMETER_EMPTY;
@@ -1439,26 +1347,6 @@ public class CnMessage extends Message {
return "不能同时强制下推和击中模型，参数 “forcedToPushDown” 和 “forced_to_index” 不能同时使用。请检查后重试。";
}
- @Override
- public String getSecondStorageNodeNotAvailable() {
- return "无法添加节点。节点不存在或被其他项目占用，请修改后重试";
- }
-
- @Override
- public String getBaseTableIndexNotAvailable() {
- return "当前无法开启分层存储。请先创建基础所明细索引。";
- }
-
- @Override
- public String getPartitionColumnNotAvailable() {
- return "当前无法开启分层存储。请将时间分区列添加到维度，并更新基础明细索引。";
- }
-
- @Override
- public String getProjectLocked() {
- return "当前项目的分层存储中正在进行数据迁移，请稍后重试。";
- }
-
@Override
public String getFixStreamingSegment() {
return "无法修复流数据模型的 Segment。";
@@ -1639,26 +1527,6 @@ public class CnMessage extends Message {
return "构建火焰图任务" + TASK_TIMEOUT;
}
- @Override
- public String getSecondStorageIndexNotSupport() {
- return "Order by列和Skipping Index列不支持使用分区列";
- }
-
- @Override
- public String getSecondStorageIndexNotAllowNullable() {
- return "Order by和Skipping Index不允许列类型为Nullable。";
- }
-
- @Override
- public String getSecondStorageOrderByHasData() {
- return "在线索引已加载分层存储数据，不支持修改Order by列。如必须修改，请先清空分层存储数据";
- }
-
- @Override
- public String getSecondStorageNodeNotAvailable(String nodeName) {
- return String.format(Locale.ROOT, "分层存储节点'%s'不可用。", nodeName);
- }
-
@Override
public String getDDLUnSupported() {
return "不支持的 DDL 语法，仅支持 `create view`, `drop view`, `alter view`, `show create table` 语法";
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
index 35e9cec125..e22f46b1e0 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
@@ -24,14 +24,6 @@ import java.util.Locale;
import org.apache.kylin.common.Singletons;
public class Message {
- private static final String SECOND_STORAGE_PROJECT_ENABLED = "The project %s does not have tiered storage enabled.";
- private static final String SECOND_STORAGE_MODEL_ENABLED = "The model %s does not have tiered storage enabled.";
- private static final String SECOND_STORAGE_SEGMENT_WITHOUT_BASE_INDEX = "The base table index is missing in the segments, please add and try again.";
- private static final String SECOND_STORAGE_DELETE_NODE_FAILED = "Node %s has data, size is %d bytes";
- private static final String SECOND_STORAGE_CARDINALITY_DATATYPE_INVALID = "The datatype is invalid. Only support LowCardinality(Nullable(String)) or Nullable(String) at the moment.";
- private static final String FORCED_TO_TIEREDSTORAGE_AND_FORCETO_INDEX = "When force_to_index=ture, the query cannot pushdown when using tiered storage fails, forcedToTieredStorage=1 or conf=1 is invalid, please modify and try again";
- private static final String FORCED_TO_TIEREDSTORAGE_RETURN_ERROR = "Query failed. Tiered storage is unavailable, please fix and try again.";
- private static final String FORCED_TO_TIEREDSTORAGE_INVALID_PRARAMETER = "invalid parameters, please fix and try again.";
private static final String DISABLE_PUSHDOWN_PROMPT = "You should turn on pushdown button if you want to pushdown.";
private static final String NON_EXISTEN_MODEL = "Model %s doesn't exist. Please confirm and try again later.";
private static final String LACK_PROJECT = "Please fill in the project parameters.";
@@ -392,10 +384,6 @@ public class Message {
return "The project name \"%s\" already exists. Please rename it.";
}
- public String getProjectDropFailedSecondStorageEnabled() {
- return "Can't delete project \"%s\", please disable tiered storage firstly.";
- }
-
public String getProjectDropFailedJobsNotKilled() {
return "Can't delete project \"%s\", please discard the related job and try again.";
}
@@ -575,7 +563,7 @@ public class Message {
// Query statistics
public String getLowLevelLicenseMessage() {
- return "Tiered Storage cannot be used. Please upgrade to Kyligence Premium version if you want to use the function.";
+ return "Please upgrade to Kyligence Premium version if you want to use the function.";
}
public String getRestartNoticeMessage() {
@@ -1198,50 +1186,6 @@ public class Message {
return "Start Time,Duration,Query ID,SQL Statement,Answered by,Query Status,Query Node,Submitter,Query Message\n";
}
- public String getSecondStorageJobExists() {
- return "Can’t turn off the tiered storage at the moment. Model “%s” has an ongoing job, Please try again later.\\n";
- }
-
- public String getSecondStorageConcurrentOperate() {
- return "Another tiered storage task is running. Please try again later.";
- }
-
- public String getSecondStorageProjectJobExists() {
- return "Can’t turn off the tiered storage at the moment. Project “%s” has an ongoing job, Please try again later.\\n";
- }
-
- public String getSecondStorageProjectEnabled() {
- return SECOND_STORAGE_PROJECT_ENABLED;
- }
-
- public String getSecondStorageModelEnabled() {
- return SECOND_STORAGE_MODEL_ENABLED;
- }
-
- public String getSecondStorageSegmentWithoutBaseIndex() {
- return SECOND_STORAGE_SEGMENT_WITHOUT_BASE_INDEX;
- }
-
- public String getSecondStorageDeleteNodeFailed() {
- return SECOND_STORAGE_DELETE_NODE_FAILED;
- }
-
- public String getInvalidLowCardinalityDataType() {
- return SECOND_STORAGE_CARDINALITY_DATATYPE_INVALID;
- }
-
- public String getJobRestartFailed() {
- return "Tiered storage task doesn't support restart.\n";
- }
-
- public String getSegmentDropFailed() {
- return "Segment can't remove. There is an ongoing load data job of tiered storage. Please try again later.\n";
- }
-
- public String getJobResumeFailed() {
- return "Tiered storage task can't resume. Please try again later.\n";
- }
-
public String getJobPauseFailed() {
return "This type of task does not support pause operation.";
}
@@ -1301,39 +1245,10 @@ public class Message {
public String getCannotForceToBothPushdodwnAndIndex() {
return "Cannot force the query to pushdown and index at the same time. Only one of the parameter “forcedToPushDown“ and “forced_to_index” could be used. Please check and try again.";
}
-
- public String getForcedToTieredstorageAndForceToIndex() {
- return FORCED_TO_TIEREDSTORAGE_AND_FORCETO_INDEX;
- }
-
- public String getForcedToTieredstorageReturnError() {
- return FORCED_TO_TIEREDSTORAGE_RETURN_ERROR;
- }
-
- public String getForcedToTieredstorageInvalidParameter() {
- return FORCED_TO_TIEREDSTORAGE_INVALID_PRARAMETER;
- }
-
public String getParameterEmpty() {
return PARAMETER_EMPTY;
}
- public String getSecondStorageNodeNotAvailable() {
- return "Can't add node. The node does not exist or has been used by other project, please modify and try again.";
- }
-
- public String getBaseTableIndexNotAvailable() {
- return "Can’t turn on the tiered storage at the moment. Please add base table index first.";
- }
-
- public String getPartitionColumnNotAvailable() {
- return "Can’t turn on the tiered storage at the moment. Please add the time partition column as dimension, and update the base table index.";
- }
-
- public String getProjectLocked() {
- return "Data migration is in progress in the current project's tiered storage, please try again later.";
- }
-
public String getFixStreamingSegment() {
return "Can’t fix segment in streaming model.";
}
@@ -1482,34 +1397,6 @@ public class Message {
return String.format(Locale.ROOT, "Cannot find target segment, and missing segment id: %s", missingSegIds);
}
- public String getSecondStorageIndexNotSupport() {
- return "Partitioning columns are not supported for Order by and Skipping Index columns";
- }
-
- public String getSecondStorageIndexNotAllowNullable() {
- return "Order by / Skipping Index doesn't support nullable column type.";
- }
-
- public String getSecondStorageOrderByHasData() {
- return "The index has already loaded the tiered storage data, and does not support modifying the Order by column. If the change is necessary ,please clear the tiered storage data first.";
- }
-
- public String getSecondStorageLayoutNotExist() {
- return "Layout id %s is not exist";
- }
-
- public String getSecondStorageLayoutNotBaseTableIndex() {
- return "Layout id %s is not base table index which is equal to tiered storage.";
- }
-
- public String getSecondStorageNotSupportType(String dataType) {
- return String.format(Locale.ROOT, "Skipping Index doesn‘t support column type '%s'.", dataType);
- }
-
- public String getSecondStorageNodeNotAvailable(String nodeName) {
- return String.format(Locale.ROOT, "Tiered storage node '%s' not available.", nodeName);
- }
-
public String getDDLUnSupported() {
return DDL_UNSUPPORTED;
}
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties b/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
index 9abe80a511..9736f42edd 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
@@ -44,7 +44,7 @@ KE-010001004=Duplicate Project Name
KE-010001005=Project Name Exceed 50
KE-010001006=Failed to Create Projects
KE-010001007=Unavailable Features
-KE-010001008=Project disable multilevel partition
+KE-010001008=Project Drop Failed
KE-010002001=Model Not Exist
KE-010002002=Empty Model Name
KE-010002003=Invalid Model Name
@@ -272,7 +272,6 @@ KE-010037005=Tiered Storage Status Error
KE-010037006=Tiered Storage Data Not Exist
KE-010037007=Tiered Storage Lock Fail
KE-010037008=Data migration is in progress
-KE-010037009=Project Drop Failed
KE-010037010=Job Restart Failed
KE-010037011=Segment Drop Failed
KE-010037012=Job Resume Failed
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties b/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
index 4ca0f9bd6d..ab32c1dd14 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
@@ -270,7 +270,6 @@ KE-010037005=分层存储状态错误
KE-010037006=分层存储数据不存在
KE-010037007=分层存储加锁失败
KE-010037008=正在进行数据迁移
-KE-010037009=项目删除异常
KE-010037010=任务无法重启
KE-010037011=Segment删除失败
KE-010037012=任务无法恢复
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
index 1558ba7b4c..6f95e2cb25 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
@@ -1182,17 +1182,6 @@ class KylinConfigBaseTest {
Assertions.assertEquals(0, config.getClusterManagerHealCheckIntervalSecond());
}
- @Test
- void testJobSchedulerMode() {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
-
- Assertions.assertEquals("DAG", config.getJobSchedulerMode());
- config.setProperty("kylin.engine.job-scheduler-mode", "CHAIN");
- Assertions.assertEquals("CHAIN", config.getJobSchedulerMode());
- config.setProperty("kylin.engine.job-scheduler-mode", "DAG");
- Assertions.assertEquals("DAG", config.getJobSchedulerMode());
- }
-
@Test
void testGetRoutineOpsTaskTimeOut() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobInfoDao.java b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobInfoDao.java
index dce995d971..e52e254540 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/dao/JobInfoDao.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/dao/JobInfoDao.java
@@ -176,8 +176,6 @@ public class JobInfoDao {
} else if (JobTypeEnum.SNAPSHOT_REFRESH == executablePO.getJobType()
|| JobTypeEnum.SNAPSHOT_BUILD == executablePO.getJobType()) {
subject = executablePO.getParams().get(NBatchConstants.P_TABLE_NAME);
- } else if (JobTypeEnum.SECOND_STORAGE_NODE_CLEAN == executablePO.getJobType()) {
- subject = jobInfo.getProject();
} else if (null != executablePO.getTargetModel() && null != executablePO.getProject()) {
if (executablePO.getParams().containsKey(NBatchConstants.P_MODEL_NAME)) {
subject = executablePO.getParams().get(NBatchConstants.P_MODEL_NAME);
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java
index b5348fc8e0..4d15151caf 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/DefaultExecutableOnModel.java
@@ -27,7 +27,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.StringHelper;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
-import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.job.model.JobParam;
import org.apache.kylin.metadata.cube.model.NBatchConstants;
import org.apache.kylin.metadata.cube.model.NDataSegment;
@@ -175,19 +174,4 @@ public class DefaultExecutableOnModel extends DefaultExecutable {
}
}
- protected static void initResourceDetectDagNode(AbstractExecutable resourceDetect, AbstractExecutable indexStep,
- AbstractExecutable secondStorage) {
- if (resourceDetect != null) {
- val indexStepId = indexStep.getId();
- indexStep.setPreviousStep(resourceDetect.getId());
- if (secondStorage != null) {
- val secondStorageId = secondStorage.getId();
- resourceDetect.setNextSteps(Sets.newHashSet(indexStepId, secondStorageId));
- secondStorage.setPreviousStep(resourceDetect.getId());
- } else {
- resourceDetect.setNextSteps(Sets.newHashSet(indexStepId));
- }
- }
- }
-
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/JobTypeEnum.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/JobTypeEnum.java
index f0d364865c..a102f918bb 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/JobTypeEnum.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/JobTypeEnum.java
@@ -40,13 +40,6 @@ public enum JobTypeEnum {
ASYNC_QUERY(Category.ASYNC_QUERY), //
- EXPORT_TO_SECOND_STORAGE(Category.SECOND_STORAGE), //
- SECOND_STORAGE_MODEL_CLEAN(Category.SECOND_STORAGE), //
- SECOND_STORAGE_NODE_CLEAN(Category.SECOND_STORAGE), //
- SECOND_STORAGE_SEGMENT_CLEAN(Category.SECOND_STORAGE), //
- SECOND_STORAGE_INDEX_CLEAN(Category.SECOND_STORAGE), //
- SECOND_STORAGE_REFRESH_SECONDARY_INDEXES(Category.SECOND_STORAGE), //
-
TABLE_SAMPLING(Category.OTHER), STAGE(Category.OTHER);
@Getter
@@ -60,7 +53,6 @@ public enum JobTypeEnum {
public static final String BUILD = "BUILD";
public static final String SNAPSHOT = "SNAPSHOT";
public static final String STREAMING = "STREAMING";
- public static final String SECOND_STORAGE = "SECOND_STORAGE";
public static final String ASYNC_QUERY = "ASYNC_QUERY";
public static final String OTHER = "OTHER";
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/manager/JobManager.java b/src/core-job/src/main/java/org/apache/kylin/job/manager/JobManager.java
index 269d634f3c..e177f2bada 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/manager/JobManager.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/manager/JobManager.java
@@ -168,8 +168,6 @@ public class JobManager {
case SNAPSHOT_REFRESH:
handler = new SnapshotJobHandler();
break;
- case EXPORT_TO_SECOND_STORAGE:
- throw new UnsupportedOperationException();
default:
log.error("jobParam doesn't have matched job: {}", jobParam.getJobTypeEnum());
return null;
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/model/JobParam.java b/src/core-job/src/main/java/org/apache/kylin/job/model/JobParam.java
index df703cb526..7c9c3d46e6 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/model/JobParam.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/model/JobParam.java
@@ -89,7 +89,10 @@ public class JobParam {
private Set<LayoutEntity> deleteLayouts;
- private Set<Long> secondStorageDeleteLayoutIds;
+ /**
+ * Used only for {@link JobTypeEnum#INDEX_BUILD} job
+ */
+ private boolean layoutsDeletableAfterBuild = false;
public JobParam addExtParams(String key, String value) {
Map<String, String> params = getExtParams();
@@ -120,7 +123,7 @@ public class JobParam {
}
public JobParam(Set<String> targetSegments, Set<Long> targetLayouts, String model, String owner,
- Set<Long> targetPartitions, Set<JobBucket> targetBuckets) {
+ Set<Long> targetPartitions, Set<JobBucket> targetBuckets) {
this(model, owner);
this.withTargetSegments(targetSegments);
this.setTargetLayouts(targetLayouts);
@@ -207,12 +210,6 @@ public class JobParam {
}
}
- public void setSecondStorageDeleteLayoutIds(Set<Long> secondStorageDeleteLayoutIds) {
- if (Objects.nonNull(secondStorageDeleteLayoutIds)) {
- this.secondStorageDeleteLayoutIds = secondStorageDeleteLayoutIds;
- }
- }
-
public String getSegment() {
if (targetSegments.size() != 1) {
return null;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
index 0f052bc748..75e5ffb030 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/model/IndexPlan.java
@@ -50,7 +50,6 @@ import org.apache.kylin.common.persistence.MissingRootPersistentEntity;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
import org.apache.kylin.guava30.shaded.common.base.Joiner;
import org.apache.kylin.guava30.shaded.common.base.Preconditions;
import org.apache.kylin.guava30.shaded.common.collect.BiMap;
@@ -669,8 +668,7 @@ public class IndexPlan extends RootPersistentEntity implements Serializable, IEn
return getIndexesMap(toBeDeletedIndexes);
... 8274 lines suppressed ...