You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/08/02 02:07:49 UTC
[kylin] branch kylin5 updated: KYLIN-5217 Remove meaningless comment and upgrade maven dependency
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 375f0e1fa3 KYLIN-5217 Remove meaningless comment and upgrade maven dependency
375f0e1fa3 is described below
commit 375f0e1fa3eea51e648763a5346a2ae82f4da0e2
Author: XiaoxiangYu <xx...@apache.org>
AuthorDate: Tue Aug 2 09:39:10 2022 +0800
KYLIN-5217 Remove meaningless comment and upgrade maven dependency
- Don't need to initialize distributed lock when running in local mode
- Add one mode of spring.profiles.active, dev,reuse, to reuse metadata stored in local file system
- Update dependency version for the integration of DataFusion
- Make QueryEngine for SparderQueryPlanExec configurable
Co-authored-by: yangzhong <ya...@ebay.com>
---
build/sbin/prepare-hadoop-conf-dir.sh | 2 +-
pom.xml | 107 ++++++---------------
.../apache/kylin/rest/KylinPrepareEnvListener.java | 15 ++-
.../apache/kylin/rest/config/AppInitializer.java | 5 +-
.../org/apache/kylin/common/KylinConfigBase.java | 6 +-
.../org/apache/kylin/job/lock/ZookeeperUtil.java | 2 -
.../kylin/metadata/project/ProjectInstance.java | 1 -
.../apache/kylin/metadata/query/QueryHistory.java | 3 +-
.../apache/kylin/metadata/query/QueryMetrics.java | 2 +-
.../DoubleQuotePushDownConverterTest.java | 2 +-
.../apache/kylin/rest/service/ModelService.java | 2 +-
.../kylin/rest/service/ModelServiceTest.java | 2 +-
.../org/apache/kylin/query/util/KapQueryUtil.java | 2 +-
.../query/util/RestoreFromComputedColumn.java | 2 +-
.../rest/controller/NQueryControllerTest.java | 2 +-
.../apache/kylin/rest/service/QueryService.java | 6 +-
.../org/apache/kylin/rest/util/SparderUIUtil.java | 1 -
.../kylin/query/engine/AsyncQueryApplication.java | 4 +-
.../engine/exec/sparder/SparderQueryPlanExec.java | 12 ++-
.../kylin/job/SecondStorageJobParamUtil.java | 2 +-
.../rest/metrics/QueryMetricsContextTest.java | 1 -
.../spark/merger/SparkJobMetadataMerger.java | 2 +-
.../engine/spark/builder/SegmentFlatTable.scala | 2 +-
.../job/stage/build/FlatTableAndDictBase.scala | 2 +-
.../engine/spark/source/NSparkSourceTest.java | 1 -
.../scala/org/apache/spark/sql/KylinSession.scala | 1 -
.../apache/spark/sql/udf/TimestampDiffTest.scala | 2 +-
src/streaming/pom.xml | 1 -
28 files changed, 75 insertions(+), 117 deletions(-)
diff --git a/build/sbin/prepare-hadoop-conf-dir.sh b/build/sbin/prepare-hadoop-conf-dir.sh
index 0dfd06bbd0..fa35b5a587 100755
--- a/build/sbin/prepare-hadoop-conf-dir.sh
+++ b/build/sbin/prepare-hadoop-conf-dir.sh
@@ -195,7 +195,7 @@ function checkExactlyAndCopyFile() {
fi
}
-# KE-9142 FI hive-site.xml is lack of some configuration
+# FI hive-site.xml is lack of some configuration
function checkAndCopyFIHiveSite() {
checkAndCopyFile $FI_ENV_PLATFORM/Hive/config/hive-site.xml
diff --git a/pom.xml b/pom.xml
index 346cd65947..dd6c8188fa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -162,7 +162,7 @@
<groovy-all.version>2.4.7</groovy-all.version>
<slf4j-ext.version>1.7.26</slf4j-ext.version>
- <protobuf-java.version>3.16.1</protobuf-java.version>
+ <protobuf-java.version>3.17.3</protobuf-java.version>
<!-- Sonar -->
<sonar.scala.version>${scala.version}</sonar.scala.version>
@@ -348,7 +348,6 @@
<module>src/data-loading-server</module>
<module>src/query-server</module>
- <!-- yinglong server (starter) -->
<module>src/data-loading-booter</module>
<module>src/query-booter</module>
<module>src/common-booter</module>
@@ -356,12 +355,6 @@
<dependencyManagement>
<dependencies>
- <!-- KAP, same order with modules -->
- <!-- <dependency>-->
- <!-- <groupId>io.kyligence.ke</groupId>-->
- <!-- <artifactId>kap-it</artifactId>-->
- <!-- <version>${project.version}</version>-->
- <!-- </dependency>-->
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-engine-spark</artifactId>
@@ -382,11 +375,6 @@
<artifactId>kylin-datasource-sdk</artifactId>
<version>${project.version}</version>
</dependency>
- <!-- <dependency>-->
- <!-- <groupId>io.kyligence.ke</groupId>-->
- <!-- <artifactId>kap-smart</artifactId>-->
- <!-- <version>${project.version}</version>-->
- <!-- </dependency>-->
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-external-catalog-sdk</artifactId>
@@ -469,11 +457,6 @@
<artifactId>kylin-datasource-service</artifactId>
<version>${project.version}</version>
</dependency>
- <!-- <dependency>-->
- <!-- <groupId>io.kyligence.yinglong</groupId>-->
- <!-- <artifactId>yinglong-enterprise-service</artifactId>-->
- <!-- <version>${project.version}</version>-->
- <!-- </dependency>-->
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-job-service</artifactId>
@@ -489,21 +472,11 @@
<artifactId>kylin-query-service</artifactId>
<version>${project.version}</version>
</dependency>
- <!-- <dependency>-->
- <!-- <groupId>io.kyligence.yinglong</groupId>-->
- <!-- <artifactId>yinglong-smart-booter</artifactId>-->
- <!-- <version>${project.version}</version>-->
- <!-- </dependency>-->
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-common-booter</artifactId>
<version>${project.version}</version>
</dependency>
- <!-- <dependency>-->
- <!-- <groupId>io.kyligence.yinglong</groupId>-->
- <!-- <artifactId>yinglong-smart-server</artifactId>-->
- <!-- <version>${project.version}</version>-->
- <!-- </dependency>-->
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-data-loading-server</artifactId>
@@ -534,11 +507,6 @@
<artifactId>kylin-metadata-server</artifactId>
<version>${project.version}</version>
</dependency>
- <!-- <dependency>-->
- <!-- <groupId>io.kyligence.yinglong</groupId>-->
- <!-- <artifactId>yinglong-smart-service</artifactId>-->
- <!-- <version>${project.version}</version>-->
- <!-- </dependency>-->
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-streaming-service</artifactId>
@@ -670,21 +638,6 @@
<artifactId>kylin-server-base</artifactId>
<version>${project.version}</version>
</dependency>
- <!-- <dependency>-->
- <!-- <groupId>io.kyligence.ke</groupId>-->
- <!-- <artifactId>kap-license-service</artifactId>-->
- <!-- <version>${project.version}</version>-->
- <!-- </dependency>-->
- <!-- <dependency>-->
- <!-- <groupId>io.kyligence.ke</groupId>-->
- <!-- <artifactId>kap-core-metadata-ext</artifactId>-->
- <!-- <version>${project.version}</version>-->
- <!-- </dependency>-->
- <!-- <dependency>-->
- <!-- <groupId>io.kyligence.ke</groupId>-->
- <!-- <artifactId>kap-query-ext</artifactId>-->
- <!-- <version>${project.version}</version>-->
- <!-- </dependency>-->
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-tool</artifactId>
@@ -750,12 +703,6 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
- <!-- <dependency>-->
- <!-- <groupId>io.kyligence.ke</groupId>-->
- <!-- <artifactId>kap-core-metadata-ext</artifactId>-->
- <!-- <version>${project.version}</version>-->
- <!-- <type>test-jar</type>-->
- <!-- </dependency>-->
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-storage</artifactId>
@@ -798,12 +745,6 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
- <!-- <dependency>-->
- <!-- <groupId>io.kyligence.ke</groupId>-->
- <!-- <artifactId>kap-it</artifactId>-->
- <!-- <version>${project.version}</version>-->
- <!-- <type>test-jar</type>-->
- <!-- </dependency>-->
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-common-service</artifactId>
@@ -1633,6 +1574,10 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -1640,6 +1585,16 @@
<artifactId>spark-catalyst_2.12</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-netty</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
@@ -1652,6 +1607,10 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -1729,6 +1688,16 @@
<version>${spark.version}</version>
<type>test-jar</type>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-netty</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
@@ -3046,24 +3015,6 @@
</build>
<profiles>
- <profile>
- <id>kyoffice</id>
-
- <repositories>
- <repository>
- <id>kyoffice</id>
- <name>Central Repository</name>
- <url>http://repo-ofs.kyligence.com:8081/repository/maven-public/</url>
- <layout>default</layout>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- </snapshots>
- </repository>
- </repositories>
- </profile>
<profile>
<id>sandbox</id>
<activation>
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
index 673b125e0b..0a30035b81 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/KylinPrepareEnvListener.java
@@ -62,7 +62,11 @@ public class KylinPrepareEnvListener implements EnvironmentPostProcessor, Ordere
setSandboxEnvs("../examples/test_case_data/sandbox");
}
} else if (env.acceptsProfiles(Profiles.of("dev"))) {
- setLocalEnvs();
+ if (env.acceptsProfiles(Profiles.of("reuse"))) {
+ setLocalEnvs(System.getProperty("kylin.test.metadata.reuse-dir"));
+ } else {
+ setLocalEnvs();
+ }
}
// enable CC check
Unsafe.setProperty("needCheckCC", "true");
@@ -105,9 +109,12 @@ public class KylinPrepareEnvListener implements EnvironmentPostProcessor, Ordere
}
private static void setLocalEnvs() {
- String tempMetadataDir = TempMetadataBuilder.prepareLocalTempMetadata();
- KylinConfig.setKylinConfigForLocalTest(tempMetadataDir);
- File localMetadata = new File(tempMetadataDir);
+ setLocalEnvs(TempMetadataBuilder.prepareLocalTempMetadata());
+ }
+
+ private static void setLocalEnvs(String metadataDir) {
+ KylinConfig.setKylinConfigForLocalTest(metadataDir);
+ File localMetadata = new File(metadataDir);
// pass checkHadoopHome
Unsafe.setProperty("hadoop.home.dir", localMetadata.getAbsolutePath() + "/working-dir");
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
index 5f698d2675..8b5216a4c9 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/AppInitializer.java
@@ -124,7 +124,10 @@ public class AppInitializer {
resourceStore.getMetadataStore().setEpochStore(epochStore);
}
- kylinConfig.getDistributedLockFactory().initialize();
+ // Don't need to initialize distributed lock when running in local mode
+ if (!kylinConfig.isUTEnv()) {
+ kylinConfig.getDistributedLockFactory().initialize();
+ }
warmUpSystemCache();
event.getApplicationContext().publishEvent(new AfterMetadataReadyEvent(event.getApplicationContext()));
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8f66b83c48..e369f693f5 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -309,7 +309,6 @@ public abstract class KylinConfigBase implements Serializable {
setProperty("kylin.log.spark-driver-properties-file", getLogSparkDriverPropertiesFile());
setProperty("kylin.log.spark-appmaster-properties-file", getLogSparkAppMasterPropertiesFile());
- // https://github.com/kyligence/kap/issues/12654
this.properties.put(WORKING_DIR_PROP,
makeQualified(new Path(this.properties.getProperty(WORKING_DIR_PROP, KYLIN_ROOT))).toString());
if (this.properties.getProperty(DATA_WORKING_DIR_PROP) != null) {
@@ -1603,6 +1602,10 @@ public abstract class KylinConfigBase implements Serializable {
// QUERY
// ============================================================================
+ public String getQueryEngineClass() {
+ return getOptional("kylin.query.query-engine-class", "org.apache.kylin.query.runtime.SparkEngine");
+ }
+
public boolean partialMatchNonEquiJoins() {
return Boolean.parseBoolean(getOptional("kylin.query.match-partial-non-equi-join-model", FALSE));
}
@@ -1697,7 +1700,6 @@ public abstract class KylinConfigBase implements Serializable {
}
// If return empty result for select star query
- // https://olapio.atlassian.net/browse/KE-23663
public boolean getEmptyResultForSelectStar() {
return Boolean.parseBoolean(getOptional("kylin.query.return-empty-result-on-select-star", "false"));
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/lock/ZookeeperUtil.java b/src/core-job/src/main/java/org/apache/kylin/job/lock/ZookeeperUtil.java
index e450995f17..c4797b072f 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/lock/ZookeeperUtil.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/lock/ZookeeperUtil.java
@@ -41,8 +41,6 @@ import org.apache.kylin.common.annotation.ThirdPartyDependencies;
/**
* DO NOT DELETE.
- *
- * Used By Kyligence Cloud.
*/
@ThirdPartyDependencies({ @ThirdPartyDependencies.ThirdPartyDependent(repository = "static-user-manager", classes = {
"AuthenticationClient" }) })
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
index ff7c696909..0e1ca7d81e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
@@ -153,7 +153,6 @@ public class ProjectInstance extends RootPersistentEntity implements ISourceAwar
}
public void initConfig(KylinConfig config) {
- // https://olapio.atlassian.net/browse/KE-11535
// to compatible with existing model
// if expose-computed-column is empty, set it with the maintainModelType
if (!overrideKylinProps.containsKey(EXPOSE_COMPUTED_COLUMN_CONF)) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistory.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistory.java
index c24f1ae908..da8672e66c 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistory.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistory.java
@@ -105,7 +105,6 @@ public class QueryHistory {
// this field is composed of modelId, layout id and index type
// it's written as modelId#layoutId#indexType
- // This way to serialized query realizations had been deprecated. See KE-20697
private String queryRealizations;
@JsonProperty(QUERY_SERVER)
@@ -215,7 +214,7 @@ public class QueryHistory {
return realizations;
}
- // This way to serialized query realizations had been deprecated. See KE-20697
+ // This way to serialized query realizations had been deprecated.
// Just for compatibility with previous versions
public List<NativeQueryRealization> transformStringRealizations() {
List<NativeQueryRealization> realizations = Lists.newArrayList();
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java
index 09a3fc1c83..656a2529e7 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java
@@ -52,7 +52,7 @@ public class QueryMetrics extends SchedulerEventNotifier {
protected String projectName;
protected String sql;
- // KE-36662 Using sql_pattern as normalized_sql storage
+ // Using sql_pattern as normalized_sql storage
protected String sqlPattern;
protected String submitter;
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/DoubleQuotePushDownConverterTest.java b/src/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/DoubleQuotePushDownConverterTest.java
index dd3baddd63..bb9486a102 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/DoubleQuotePushDownConverterTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/DoubleQuotePushDownConverterTest.java
@@ -84,7 +84,7 @@ public class DoubleQuotePushDownConverterTest extends NLocalFileMetadataTestCase
Arrays.asList("select a + b * c", "select \"A\" + \"B\" * \"C\""),
Arrays.asList("select 1 + b * c", "select 1 + \"B\" * \"C\""),
- //for filter the function without parentheses,ref KE-13407
+ //for filter the function without parentheses
Arrays.asList(
"select CURRENT_CATALOG, CURRENT_DATE, CURRENT_PATH,CURRENT_ROLE, CURRENT_SCHEMA, CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_USER,LOCALTIME, LOCALTIMESTAMP, SESSION_USER, SYSTEM_USER, USER",
"select CURRENT_CATALOG, CURRENT_DATE, CURRENT_PATH,CURRENT_ROLE, CURRENT_SCHEMA, CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_USER,LOCALTIME, LOCALTIMESTAMP, SESSION_USER, SYSTEM_USER, USER"),
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 5afe75d31a..3ba0d4c5c0 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -3736,7 +3736,7 @@ public class ModelService extends BasicService implements TableModelSupporter, P
/**
* Validate computed column type and throw errors to report wrongly typed computed columns.
- * Models migrated from 3x may have wrongly typed computed columns. see KE-11862
+ * Models migrated from 3x may have wrongly typed computed columns.
*
* @param modelId
* @param project
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 3d7631962d..4feffb4314 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -677,7 +677,7 @@ public class ModelServiceTest extends SourceTestCase {
Assert.assertEquals(3, segments.size());
Assert.assertEquals("MERGING", segments.get(2).getStatusToDisplay().toString());
- // KE-25547, complete segment response
+ // complete segment response
val seg2Resp = segments.stream().filter(s -> s.getId().equals(seg2.getId())).findFirst().get();
Assert.assertNotNull(seg2Resp);
Assert.assertEquals(seg2.isSnapshotReady(), seg2Resp.isSnapshotReady());
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/KapQueryUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/KapQueryUtil.java
index 1472d64f4b..7e835fe96e 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/KapQueryUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/KapQueryUtil.java
@@ -394,7 +394,7 @@ public class KapQueryUtil {
String sql = queryParams.getSql();
while (sql.endsWith(";"))
sql = sql.substring(0, sql.length() - 1);
- // fix KE-34379，filter "/*+ MODEL_PRIORITY({cube_name}) */" hint
+ // fix，filter "/*+ MODEL_PRIORITY({cube_name}) */" hint
String regex = "/\\*\\s*\\+\\s*(?i)MODEL_PRIORITY\\s*\\([\\s\\S]*\\)\\s*\\*/";
sql = Pattern.compile(regex).matcher(sql).replaceAll("");
initPushDownConvertersIfNeeded(queryParams.getKylinConfig());
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java b/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java
index e569944c91..9dc19d8c7e 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java
@@ -260,7 +260,7 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
// however user query might use a different alias, say bc.x + ba.y
String ccExpression = CalciteParser.replaceAliasInExpr(computedColumnDesc.getExpression(),
matchInfo.getAliasMapping().inverse());
- // intend to handle situation like KE-15939
+ // intend to handle situation
String replaceExpression = columnUsage.addAlias ? ccExpression + " AS " + computedColumnDesc.getColumnName()
: ccExpression;
diff --git a/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java b/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
index 0a3d1793bf..40007ce7fd 100644
--- a/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
+++ b/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
@@ -527,7 +527,7 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
.andExpect(MockMvcResultMatchers.status().is(400));
}
- //KE-22624 public query history api
+ // public query history api
@Test
public void testGetQueryHistoriesAPI() throws Exception {
QueryHistoryRequest request = new QueryHistoryRequest();
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 7290ea4585..336539b546 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -724,9 +724,9 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
try {
if (!sqlResponse.isPrepare() && QueryMetricsContext.isStarted()) {
val queryMetricsContext = QueryMetricsContext.collect(QueryContext.current());
- // KE-35556 Set stored sql a structured format json string
+ // Set stored sql a structured format json string
queryMetricsContext.setSql(constructQueryHistorySqlText(sqlRequest, sqlResponse, originalSql));
- // KE-36662 Using sql_pattern as normalized_sql storage
+ // Using sql_pattern as normalized_sql storage
String normalizedSql = QueryContext.currentMetrics().getCorrectedSql();
queryMetricsContext.setSqlPattern(normalizedSql);
QueryHistoryScheduler queryHistoryScheduler = QueryHistoryScheduler.getInstance();
@@ -755,7 +755,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
}
}
- // KE-36662 Do not store normalized_sql in sql_text, as it may exceed storage limitation
+ // Do not store normalized_sql in sql_text, as it may exceed storage limitation
return QueryHistoryUtil.toQueryHistorySqlText(new QueryHistorySql(originalSql, null, params));
}
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/util/SparderUIUtil.java b/src/query-service/src/main/java/org/apache/kylin/rest/util/SparderUIUtil.java
index 7f45f90835..a1ea9dba30 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/util/SparderUIUtil.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/util/SparderUIUtil.java
@@ -96,7 +96,6 @@ public class SparderUIUtil {
return;
}
- // KE-12678
proxyBase = "/proxy/" + ui.appId();
// reset
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryApplication.java b/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryApplication.java
index 01fdf5e3ab..4810b3892c 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryApplication.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/AsyncQueryApplication.java
@@ -98,7 +98,7 @@ public class AsyncQueryApplication extends SparkApplication {
try {
QueryMetricsContext queryMetricsContext = QueryMetricsContext.collect(queryContext);
queryMetricsContext.setSql(constructQueryHistorySqlText(queryParams, queryContext.getUserSQL()));
- // KE-36662 Using sql_pattern as normalized_sql storage
+ // Using sql_pattern as normalized_sql storage
String normalizedSql = QueryContext.currentMetrics().getCorrectedSql();
queryMetricsContext.setSqlPattern(normalizedSql);
@@ -123,7 +123,7 @@ public class AsyncQueryApplication extends SparkApplication {
}
}
- // KE-36662 Do not store normalized_sql in sql_text, as it may exceed storage limitation
+ // Do not store normalized_sql in sql_text, as it may exceed storage limitation
return QueryHistoryUtil.toQueryHistorySqlText(new QueryHistorySql(originalSql, null, params));
}
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
index a9a9963ae4..e1566ea0a0 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
@@ -32,6 +32,7 @@ import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.QueryErrorCode;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.relnode.OLAPRel;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
@@ -43,10 +44,10 @@ import org.apache.kylin.query.engine.meta.SimpleDataContext;
import org.apache.kylin.query.relnode.ContextUtil;
import org.apache.kylin.query.relnode.KapContext;
import org.apache.kylin.query.relnode.KapRel;
-import org.apache.kylin.query.runtime.SparkEngine;
import org.apache.kylin.query.util.QueryContextCutter;
import org.apache.spark.SparkException;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -84,9 +85,12 @@ public class SparderQueryPlanExec implements QueryPlanExec {
}
}
+ Preconditions.checkArgument(dataContext instanceof SimpleDataContext);
+ SimpleDataContext sparderDataContext = (SimpleDataContext) dataContext;
+
// rewrite
rewrite(rel);
- return doExecute(rel, dataContext);
+ return doExecute(rel, sparderDataContext);
}
/**
@@ -95,8 +99,8 @@ public class SparderQueryPlanExec implements QueryPlanExec {
* @param dataContext
* @return
*/
- private ExecuteResult doExecute(RelNode rel, DataContext dataContext) {
- QueryEngine queryEngine = new SparkEngine();
+ private ExecuteResult doExecute(RelNode rel, SimpleDataContext dataContext) {
+ QueryEngine queryEngine = (QueryEngine) ClassUtil.newInstance(dataContext.getKylinConfig().getQueryEngineClass());
return internalCompute(queryEngine, dataContext, rel.getInput(0));
}
diff --git a/src/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageJobParamUtil.java b/src/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageJobParamUtil.java
index 90cc7ab42a..28660987eb 100644
--- a/src/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageJobParamUtil.java
+++ b/src/second-storage/core/src/main/java/org/apache/kylin/job/SecondStorageJobParamUtil.java
@@ -74,7 +74,7 @@ public class SecondStorageJobParamUtil {
/**
* build delete layout table parameters
*
- * PRD_KE-34597 add index clean job
+ * add index clean job
*
* @param project project name
* @param model model id
diff --git a/src/server-base/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsContextTest.java b/src/server-base/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsContextTest.java
index 806c0298be..2137d7cbf4 100644
--- a/src/server-base/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsContextTest.java
+++ b/src/server-base/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsContextTest.java
@@ -360,7 +360,6 @@ public class QueryMetricsContextTest extends NLocalFileMetadataTestCase {
@Test
public void testWhenHitStorageCache() {
- //this is for https://olapio.atlassian.net/browse/KE-12573
final String origSql = "select * from test_parse_sql_pattern_error;";
final String massagedSql = "select * from test_parse_sql_pattern_error";
final String sqlPattern = "SELECT *\n" + "FROM \"TEST_PARSE_SQL_PATTERN_ERROR\"";
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/SparkJobMetadataMerger.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/SparkJobMetadataMerger.java
index e9cf08bec7..2e1b1b4d22 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/SparkJobMetadataMerger.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/SparkJobMetadataMerger.java
@@ -175,7 +175,7 @@ public abstract class SparkJobMetadataMerger extends MetadataMerger {
item.forEach((k, v) -> //
merged.put(k, v + merged.getOrDefault(k, 0L))));
localSegment.setColumnSourceBytes(merged);
- // KE-18417 snapshot management.
+ // snapshot management.
localSegment.setLastBuildTime(newSegment.getLastBuildTime());
localSegment.setSourceBytesSize(newSegment.getSourceBytesSize());
localSegment.setLastBuildTime(lastBuildTime);
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
index 8eefca1a64..c045b11301 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
@@ -301,7 +301,7 @@ class SegmentFlatTable(private val sparkSession: SparkSession, //
DFBuilderHelper.checkPointSegment(dataSegment, (copied: NDataSegment) => {
copied.setFlatTableReady(true)
if (dataSegment.isFlatTableReady) {
- // KE-14714 if flat table is updated, there might be some data inconsistency across indexes
+ // if flat table is updated, there might be some data inconsistency across indexes
copied.setStatus(SegmentStatusEnum.WARNING)
}
})
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
index ce5e56d8ef..5af86a9d90 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
@@ -309,7 +309,7 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
DFBuilderHelper.checkPointSegment(dataSegment, (copied: NDataSegment) => {
copied.setFlatTableReady(true)
if (dataSegment.isFlatTableReady) {
- // KE-14714 if flat table is updated, there might be some data inconsistency across indexes
+ // if flat table is updated, there might be some data inconsistency across indexes
copied.setStatus(SegmentStatusEnum.WARNING)
}
})
diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/source/NSparkSourceTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/source/NSparkSourceTest.java
index 5213f0416b..7f2450cf9a 100644
--- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/source/NSparkSourceTest.java
+++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/source/NSparkSourceTest.java
@@ -57,7 +57,6 @@ public class NSparkSourceTest extends NLocalWithSparkSessionTest {
}
/**
- * for the issue: https://olapio.atlassian.net/browse/KE-9497
*/
@Test
public void testGetSourceData() {
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KylinSession.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KylinSession.scala
index c36161ee95..4ea13d1949 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KylinSession.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KylinSession.scala
@@ -167,7 +167,6 @@ object KylinSession extends Logging {
// maybe this is an existing SparkContext, update its SparkConf which maybe used
// by SparkSession
- // KE-12678
if (sc.master.startsWith("yarn")) {
Unsafe.setProperty("spark.ui.proxyBase", "/proxy/" + sc.applicationId)
}
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/udf/TimestampDiffTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/udf/TimestampDiffTest.scala
index c594160459..8949dc358f 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/udf/TimestampDiffTest.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/spark/sql/udf/TimestampDiffTest.scala
@@ -56,7 +56,7 @@ class TimestampDiffTest extends SparderBaseFunSuite with SharedSparkSession with
// DAY
verifyResult("select timestampdiff('DAY', date'2016-02-01' , date'2016-02-01')", Seq("0"))
verifyResult("select timestampdiff('DAY', date'2016-02-01' , date'2016-02-02')", Seq("1"))
- // TODO KE-27654
+ // TODO
// verifyResult("select timestampdiff('DAY', date'1977-04-20', date'1987-08-02')", Seq("3756"))
// HOUR
diff --git a/src/streaming/pom.xml b/src/streaming/pom.xml
index d7154f1320..62bfd0e9a8 100644
--- a/src/streaming/pom.xml
+++ b/src/streaming/pom.xml
@@ -156,7 +156,6 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<scope>test</scope>
- <version>3.4.0</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>