You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/02/14 05:54:37 UTC
[kylin] 08/33: KYLIN-5428 Module refactor
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b0c7326856cc6ef8e55212f3a383503119da15c4
Author: qianhao.zhou <z....@gmail.com>
AuthorDate: Tue Nov 29 12:08:03 2022 +0800
KYLIN-5428 Module refactor
* refactor sys-tools & kylin-tool
* fix core-storage UT
* fix kylin-tool pom
* refactor core-job
* refactor core-job
* refactor core-job
* fix code smell
* fix UT
Co-authored-by: qhzhou <qi...@kyligence.io>
---
pom.xml | 16 +-
src/common-server/pom.xml | 5 +
src/common-service/pom.xml | 215 ++++-----------------
.../org/apache/kylin/rest/cache/KylinEhCache.java | 0
.../apache/kylin/rest/security/LoginLogFilter.java | 0
.../apache/kylin/rest/cache/KylinEhCacheTest.java | 5 +-
.../apache/kylin/storage/cache/EhcacheTest.java | 0
src/common-service/src/test/resources/ehcache.xml | 2 +-
.../src/test/resources/user-defined-ehcache.xml | 0
.../org/apache/kylin/rest/cache/KylinCache.java | 0
src/core-job/pom.xml | 40 ++--
.../kylin/common/util/SystemInfoCollector.java | 6 +-
.../kylin/job/execution/AbstractExecutable.java | 4 +-
src/core-metadata/pom.xml | 33 ++--
.../cube/storage/GarbageStorageCollector.java | 0
.../cube/storage/ProjectStorageInfoCollector.java | 6 +-
.../cube/storage/StorageInfoCollector.java | 0
.../metadata/cube/storage/StorageInfoEnum.java | 0
.../cube/storage/StorageQuotaCollector.java | 0
.../metadata/cube/storage/StorageVolumeInfo.java | 0
.../cube/storage/TotalStorageCollector.java | 0
.../metadata/query/CuboidLayoutQueryTimes.java | 6 -
.../kylin/metadata/query/QueryStatistics.java | 12 +-
.../kylin/metadata/query/QueryTimesResponse.java | 3 -
.../apache/kylin/metrics/HdfsCapacityMetrics.java | 0
.../storage/ProjectStorageInfoCollectorTest.java | 0
.../kylin/metrics/HdfsCapacityMetricsTest.java | 0
src/core-metrics/pom.xml | 5 -
src/core-storage/pom.xml | 12 +-
src/data-loading-server/pom.xml | 4 +
src/datasource-service/pom.xml | 9 +-
src/kylin-it/pom.xml | 7 +-
.../apache/kylin/newten/CalciteDynamicTest.java | 5 +-
.../kylin/newten/NBuildAndQuerySnapshotTest.java | 5 +-
.../org/apache/kylin/newten/TimeZoneQueryTest.java | 17 +-
src/metadata-server/pom.xml | 4 +
src/query-server/pom.xml | 11 ++
src/query-service/pom.xml | 10 -
.../rest/health/SparkContextHealthIndicator.java | 3 +-
.../health/SparkSqlContextHealthIndicator.java | 3 +-
.../kylin/rest/service/AysncQueryServiceTest.java | 3 +-
src/second-storage/core-ui/pom.xml | 5 +
src/second-storage/core/pom.xml | 6 -
.../kap/secondstorage/metadata/NodeGroupTest.java | 2 +-
src/server/pom.xml | 20 +-
.../rest/config/KylinRequestHandlerSelectors.java | 0
.../apache/kylin/rest/config/SecurityConfig.java | 0
.../apache/kylin/rest/config/SwaggerConfig.java | 0
.../rest/health/FileSystemHealthIndicator.java | 6 +-
.../rest/health/MetaStoreHealthIndicator.java | 34 +++-
.../rest/security/UnauthorisedEntryPoint.java | 0
src/source-hive/pom.xml | 4 -
.../kylin/engine/spark/job/NSparkExecutable.java | 4 +-
.../engine/spark/job/NSparkSnapshotJobTest.java | 5 +-
src/systools/pom.xml | 153 ++-------------
.../rest/health/AbstractKylinHealthIndicator.java | 47 -----
.../kylin/rest/security/KerberosLoginManager.java | 4 +-
.../apache/kylin/rest/util/AclPermissionUtil.java | 2 +-
.../rest/security/KerberosLoginManagerTest.java | 10 +-
.../kylin/rest/security/UserAclManagerTest.java | 2 +-
src/systools/src/test/resources/ehcache.xml | 19 --
src/tool/pom.xml | 66 +++----
.../org/apache/kylin/tool/RecCandidateTool.java | 2 +-
.../apache/kylin/tool/RecCandidateToolTest.java | 0
64 files changed, 234 insertions(+), 608 deletions(-)
diff --git a/pom.xml b/pom.xml
index 90135eb7ef..fcd1c2c27f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -594,12 +594,6 @@
<artifactId>kylin-core-metrics</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-streaming</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- </dependency>
<dependency>
<groupId>org.freemarker</groupId>
@@ -1195,11 +1189,6 @@
<version>${commons-cli.version}</version>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>${commons-lang.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
@@ -2047,6 +2036,7 @@
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.owasp.esapi</groupId>
@@ -2930,6 +2920,10 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ </dependency>
</dependencies>
<distributionManagement>
diff --git a/src/common-server/pom.xml b/src/common-server/pom.xml
index 7ac148c88f..0e23257e0a 100644
--- a/src/common-server/pom.xml
+++ b/src/common-server/pom.xml
@@ -113,6 +113,11 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-log4j2</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.springfox</groupId>
+ <artifactId>springfox-boot-starter</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
diff --git a/src/common-service/pom.xml b/src/common-service/pom.xml
index e74994f577..66c8a53bfd 100644
--- a/src/common-service/pom.xml
+++ b/src/common-service/pom.xml
@@ -35,66 +35,63 @@
<artifactId>kylin-tool</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kap-second-storage-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-streaming</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-job</artifactId>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-aop</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-tool</artifactId>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-web</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-spark-ddl</artifactId>
+ <groupId>commons-fileupload</groupId>
+ <artifactId>commons-fileupload</artifactId>
</dependency>
-
<dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-config</artifactId>
</dependency>
<dependency>
- <groupId>commons-fileupload</groupId>
- <artifactId>commons-fileupload</artifactId>
+ <groupId>redis.clients</groupId>
+ <artifactId>jedis</artifactId>
</dependency>
-
<dependency>
<groupId>net.sf.ehcache</groupId>
<artifactId>ehcache</artifactId>
</dependency>
- <dependency>
- <groupId>cglib</groupId>
- <artifactId>cglib</artifactId>
- </dependency>
- <dependency>
- <groupId>com.jayway.jsonpath</groupId>
- <artifactId>json-path</artifactId>
- </dependency>
+ <!-- Spring Boot -->
<dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-jdbc</artifactId>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-actuator</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>spring-context</artifactId>
+ <groupId>org.springframework</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-aop</artifactId>
+ <groupId>org.springframework.session</groupId>
+ <artifactId>spring-session-core</artifactId>
</dependency>
+ <!-- hadoop -->
<dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-context-support</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka-clients</artifactId>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>
+
+
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
@@ -127,142 +124,6 @@
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
- <version>${mockito.version}</version>
- </dependency>
-
- <!-- spring aop -->
- <dependency>
- <groupId>org.aspectj</groupId>
- <artifactId>aspectjrt</artifactId>
- </dependency>
- <dependency>
- <groupId>org.aspectj</groupId>
- <artifactId>aspectjweaver</artifactId>
- </dependency>
- <dependency>
- <groupId>redis.clients</groupId>
- <artifactId>jedis</artifactId>
- </dependency>
-
- <!-- Spring Boot -->
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-autoconfigure</artifactId>
- <version>${spring.boot.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>spring-context</artifactId>
- <groupId>org.springframework</groupId>
- </exclusion>
- <exclusion>
- <artifactId>spring-core</artifactId>
- <groupId>org.springframework</groupId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- database -->
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-dbcp2</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- </dependency>
-
- <dependency>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- <version>${xerces.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.glassfish.jersey.core</groupId>
- <artifactId>jersey-common</artifactId>
- <version>${jersey.version}</version>
- </dependency>
-
- <!-- Apache POI -->
- <dependency>
- <groupId>org.apache.poi</groupId>
- <artifactId>poi</artifactId>
- <version>${poi.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.poi</groupId>
- <artifactId>poi-ooxml</artifactId>
- <version>${poi.version}</version>
- </dependency>
-
- <!-- spring security -->
- <dependency>
- <groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-ext-jdk15on</artifactId>
- </dependency>
- <dependency>
- <groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk15on</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.tomcat.embed</groupId>
- <artifactId>tomcat-embed-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- <version>4.4.5</version>
- </dependency>
- <dependency>
- <groupId>org.apache.calcite.avatica</groupId>
- <artifactId>avatica-core</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-web</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.yaml</groupId>
- <artifactId>snakeyaml</artifactId>
- </dependency>
-
- <dependency>
- <groupId>javax.validation</groupId>
- <artifactId>validation-api</artifactId>
- </dependency>
-
- <!-- hadoop -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </dependency>
-
- <!-- test -->
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-job</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-systools</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
</dependency>
<!-- spark -->
@@ -272,16 +133,6 @@
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-junit-jupiter</artifactId>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
diff --git a/src/systools/src/main/java/org/apache/kylin/rest/cache/KylinEhCache.java b/src/common-service/src/main/java/org/apache/kylin/rest/cache/KylinEhCache.java
similarity index 100%
rename from src/systools/src/main/java/org/apache/kylin/rest/cache/KylinEhCache.java
rename to src/common-service/src/main/java/org/apache/kylin/rest/cache/KylinEhCache.java
diff --git a/src/systools/src/main/java/org/apache/kylin/rest/security/LoginLogFilter.java b/src/common-service/src/main/java/org/apache/kylin/rest/security/LoginLogFilter.java
similarity index 100%
rename from src/systools/src/main/java/org/apache/kylin/rest/security/LoginLogFilter.java
rename to src/common-service/src/main/java/org/apache/kylin/rest/security/LoginLogFilter.java
diff --git a/src/systools/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java
similarity index 91%
rename from src/systools/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java
rename to src/common-service/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java
index f71faaac54..9d6f4d3108 100644
--- a/src/systools/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/cache/KylinEhCacheTest.java
@@ -47,14 +47,15 @@ public class KylinEhCacheTest extends NLocalFileMetadataTestCase {
overwriteSystemProp("kylin.cache.config", "file://" + url + "not-exists-ehcache.xml");
KylinCache cache = KylinEhCache.getInstance();
CacheManager cacheManager = (CacheManager) FieldUtils.getFieldValue(cache, "cacheManager");
- Assert.assertEquals("DefaultCacheName", cacheManager.getName());
+ Assert.assertEquals("DEFAULT_CACHE", cacheManager.getName());
cacheManager.shutdown();
}
@Test
public void testRightCacheConfig() throws IllegalAccessException {
String url = Thread.currentThread().getContextClassLoader().getResource("").getPath();
- overwriteSystemProp("kylin.cache.config", "file://" + url + "user-defined-ehcache.xml");
+ String location = "file://" + url + "user-defined-ehcache.xml";
+ overwriteSystemProp("kylin.cache.config", location);
KylinCache cache = KylinEhCache.getInstance();
CacheManager cacheManager = (CacheManager) FieldUtils.getFieldValue(cache, "cacheManager");
Assert.assertEquals("UserDefinedCache", cacheManager.getName());
diff --git a/src/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java b/src/common-service/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
similarity index 100%
rename from src/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
rename to src/common-service/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
diff --git a/src/common-service/src/test/resources/ehcache.xml b/src/common-service/src/test/resources/ehcache.xml
index f7000111a5..c39e3687ea 100644
--- a/src/common-service/src/test/resources/ehcache.xml
+++ b/src/common-service/src/test/resources/ehcache.xml
@@ -15,7 +15,7 @@
limitations under the License.
-->
-<ehcache maxBytesLocalHeap="512M">
+<ehcache maxBytesLocalHeap="512M" name="DEFAULT_CACHE">
<cache name="StorageCache"
eternal="false"
timeToIdleSeconds="86400"
diff --git a/src/systools/src/test/resources/user-defined-ehcache.xml b/src/common-service/src/test/resources/user-defined-ehcache.xml
similarity index 100%
rename from src/systools/src/test/resources/user-defined-ehcache.xml
rename to src/common-service/src/test/resources/user-defined-ehcache.xml
diff --git a/src/systools/src/main/java/org/apache/kylin/rest/cache/KylinCache.java b/src/core-common/src/main/java/org/apache/kylin/rest/cache/KylinCache.java
similarity index 100%
rename from src/systools/src/main/java/org/apache/kylin/rest/cache/KylinCache.java
rename to src/core-common/src/main/java/org/apache/kylin/rest/cache/KylinCache.java
diff --git a/src/core-job/pom.xml b/src/core-job/pom.xml
index 298bc6298e..d41f67cfd2 100644
--- a/src/core-job/pom.xml
+++ b/src/core-job/pom.xml
@@ -41,40 +41,19 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-metadata</artifactId>
</dependency>
- <dependency>
- <groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-curator-test</artifactId>
- </dependency>
- <dependency>
- <groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-curator</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-metrics</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metrics</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
+ <groupId>com.github.oshi</groupId>
+ <artifactId>oshi-core</artifactId>
</dependency>
-
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
- <groupId>com.github.oshi</groupId>
- <artifactId>oshi-core</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
@@ -89,6 +68,17 @@
</dependency>
<!--Env & Test-->
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.kyligence.ke</groupId>
+ <artifactId>kap-external-curator-test</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
diff --git a/src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java b/src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java
index d8440cff37..b330a327e2 100644
--- a/src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java
+++ b/src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java
@@ -24,7 +24,6 @@ import org.apache.kylin.common.KylinConfig;
import lombok.val;
import oshi.SystemInfo;
import oshi.hardware.HardwareAbstractionLayer;
-import oshi.software.os.OperatingSystem;
public final class SystemInfoCollector {
@@ -37,16 +36,13 @@ public final class SystemInfoCollector {
private static HardwareAbstractionLayer hal = null;
- private static OperatingSystem os = null;
-
static {
init();
}
- public static void init() {
+ private static void init() {
SystemInfo si = new SystemInfo();
hal = si.getHardware();
- os = si.getOperatingSystem();
}
public static Integer getAvailableMemoryInfo() {
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index a8c8857a75..db308fc8f5 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -50,6 +50,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.stream.Collectors;
+import io.kyligence.kap.guava20.shaded.common.base.Throwables;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
@@ -88,7 +89,6 @@ import com.google.common.collect.Sets;
import io.kyligence.kap.guava20.shaded.common.annotations.VisibleForTesting;
import io.kyligence.kap.guava20.shaded.common.base.MoreObjects;
-import io.kyligence.kap.shaded.curator.org.apache.curator.shaded.com.google.common.base.Throwables;
import lombok.Getter;
import lombok.Setter;
import lombok.val;
@@ -264,7 +264,7 @@ public abstract class AbstractExecutable implements Executable {
}, project, UnitOfWork.DEFAULT_MAX_RETRY, getEpochId(), getTempLockName());
} catch (Exception e) {
if (Throwables.getCausalChain(e).stream().anyMatch(x -> x instanceof JobStoppedException)) {
- // "in this short period user might changed job state" happens
+ // "in this short period user might change job state" happens
logger.info("[LESS_LIKELY_THINGS_HAPPENED] JobStoppedException thrown from in a UnitOfWork", e);
tryAgain = true;
} else {
diff --git a/src/core-metadata/pom.xml b/src/core-metadata/pom.xml
index 80a87ca8c0..323788989a 100644
--- a/src/core-metadata/pom.xml
+++ b/src/core-metadata/pom.xml
@@ -36,18 +36,10 @@
</properties>
<dependencies>
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
</dependency>
- <dependency>
- <groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-influxdb</artifactId>
- </dependency>
<dependency>
<groupId>com.tdunning</groupId>
<artifactId>t-digest</artifactId>
@@ -76,10 +68,6 @@
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</dependency>
- <dependency>
- <groupId>net.sf.ehcache</groupId>
- <artifactId>ehcache</artifactId>
- </dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-core</artifactId>
@@ -93,16 +81,27 @@
<artifactId>mybatis-dynamic-sql</artifactId>
</dependency>
- <!--Env & Test-->
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.ehcache</groupId>
+ <artifactId>ehcache</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!--Env & Test-->
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-common</artifactId>
+ <type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java
similarity index 100%
rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java
rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/GarbageStorageCollector.java
diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
similarity index 91%
rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
index 6a5f4d7c56..ea864bf15f 100644
--- a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollector.java
@@ -31,14 +31,14 @@ public class ProjectStorageInfoCollector {
private List<StorageInfoCollector> collectors = Lists.newArrayList();
- private static final ImmutableMap<Class, StorageInfoEnum> collectorType = ImmutableMap
- .<Class, StorageInfoEnum> builder().put(GarbageStorageCollector.class, StorageInfoEnum.GARBAGE_STORAGE)
+ private static final ImmutableMap<Class<?>, StorageInfoEnum> collectorType = ImmutableMap
+ .<Class<?>, StorageInfoEnum> builder().put(GarbageStorageCollector.class, StorageInfoEnum.GARBAGE_STORAGE)
.put(TotalStorageCollector.class, StorageInfoEnum.TOTAL_STORAGE)
.put(StorageQuotaCollector.class, StorageInfoEnum.STORAGE_QUOTA).build();
public ProjectStorageInfoCollector(List<StorageInfoEnum> storageInfoList) {
if (CollectionUtils.isNotEmpty(storageInfoList)) {
- storageInfoList.forEach(si -> addCollectors(si));
+ storageInfoList.forEach(this::addCollectors);
}
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java
similarity index 100%
rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java
rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoCollector.java
diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoEnum.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoEnum.java
similarity index 100%
rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoEnum.java
rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageInfoEnum.java
diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java
similarity index 100%
rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java
rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageQuotaCollector.java
diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageVolumeInfo.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageVolumeInfo.java
similarity index 100%
rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/StorageVolumeInfo.java
rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/StorageVolumeInfo.java
diff --git a/src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java
similarity index 100%
rename from src/core-job/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java
rename to src/core-metadata/src/main/java/org/apache/kylin/metadata/cube/storage/TotalStorageCollector.java
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/CuboidLayoutQueryTimes.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/CuboidLayoutQueryTimes.java
index 22b1077dfb..3605935abb 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/CuboidLayoutQueryTimes.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/CuboidLayoutQueryTimes.java
@@ -18,20 +18,14 @@
package org.apache.kylin.metadata.query;
-import io.kyligence.kap.shaded.influxdb.org.influxdb.annotation.Column;
-import io.kyligence.kap.shaded.influxdb.org.influxdb.annotation.Measurement;
import lombok.Getter;
import lombok.Setter;
@SuppressWarnings("serial")
-@Measurement(name = "realization_metric")
@Getter
@Setter
public class CuboidLayoutQueryTimes {
- @Column(name = "model")
private String modelId;
- @Column(name = "layout_id")
private String layoutId;
- @Column(name = "query_times")
private int queryTimes;
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryStatistics.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryStatistics.java
index b819aa4905..231de32242 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryStatistics.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryStatistics.java
@@ -18,44 +18,36 @@
package org.apache.kylin.metadata.query;
-import java.time.Instant;
-
import com.fasterxml.jackson.annotation.JsonProperty;
-
-import io.kyligence.kap.shaded.influxdb.org.influxdb.annotation.Column;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
+import java.time.Instant;
+
@Getter
@Setter
@NoArgsConstructor
public class QueryStatistics {
@JsonProperty("engine_type")
- @Column(name = "engine_type", tag = true)
private String engineType;
@JsonProperty("count")
- @Column(name = "count")
private long count;
@JsonProperty("ratio")
private double ratio;
@JsonProperty("mean")
- @Column(name = "mean")
private double meanDuration;
@JsonProperty("model")
- @Column(name = "model", tag = true)
private String model;
@JsonProperty("time")
- @Column(name = "time")
private Instant time;
@JsonProperty("month")
- @Column(name = "month", tag = true)
private String month;
public QueryStatistics(String engineType) {
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryTimesResponse.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryTimesResponse.java
index e3973b2f91..333feb66ff 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryTimesResponse.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryTimesResponse.java
@@ -18,15 +18,12 @@
package org.apache.kylin.metadata.query;
-import io.kyligence.kap.shaded.influxdb.org.influxdb.annotation.Column;
import lombok.Data;
@Data
public class QueryTimesResponse {
- @Column(name = "model")
private String model;
- @Column(name = "query_times")
private int queryTimes;
}
diff --git a/src/core-job/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java b/src/core-metadata/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java
similarity index 100%
rename from src/core-job/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java
rename to src/core-metadata/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java
diff --git a/src/core-job/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java
similarity index 100%
rename from src/core-job/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java
rename to src/core-metadata/src/test/java/org/apache/kylin/metadata/cube/storage/ProjectStorageInfoCollectorTest.java
diff --git a/src/core-job/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
similarity index 100%
rename from src/core-job/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
rename to src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
diff --git a/src/core-metrics/pom.xml b/src/core-metrics/pom.xml
index fb2e75ff70..8902aae48f 100644
--- a/src/core-metrics/pom.xml
+++ b/src/core-metrics/pom.xml
@@ -67,11 +67,6 @@
<artifactId>commons-collections</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-core</artifactId>
diff --git a/src/core-storage/pom.xml b/src/core-storage/pom.xml
index 4de9e7315d..8846566e70 100644
--- a/src/core-storage/pom.xml
+++ b/src/core-storage/pom.xml
@@ -41,25 +41,15 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-metadata</artifactId>
</dependency>
-
- <!-- Provided -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
-
</dependencies>
</project>
diff --git a/src/data-loading-server/pom.xml b/src/data-loading-server/pom.xml
index b49dc21c87..7ce2077b10 100644
--- a/src/data-loading-server/pom.xml
+++ b/src/data-loading-server/pom.xml
@@ -62,6 +62,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.springfox</groupId>
+ <artifactId>springfox-boot-starter</artifactId>
+ </dependency>
<!--Env & Test-->
<dependency>
diff --git a/src/datasource-service/pom.xml b/src/datasource-service/pom.xml
index f0b8f9eb84..efdb2d3061 100644
--- a/src/datasource-service/pom.xml
+++ b/src/datasource-service/pom.xml
@@ -35,10 +35,6 @@
</properties>
<dependencies>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-systools</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-common-service</artifactId>
@@ -60,6 +56,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-spark-ddl</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
@@ -100,6 +100,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
diff --git a/src/kylin-it/pom.xml b/src/kylin-it/pom.xml
index 84db80f22f..ebeb329508 100644
--- a/src/kylin-it/pom.xml
+++ b/src/kylin-it/pom.xml
@@ -50,12 +50,7 @@
<artifactId>commons-io</artifactId>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <scope>compile</scope>
- </dependency>
-
+ <!--yinglong-->
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-query-server</artifactId>
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/CalciteDynamicTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/CalciteDynamicTest.java
index ceacaea174..98403fa32e 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/CalciteDynamicTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/CalciteDynamicTest.java
@@ -30,7 +30,6 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
import org.apache.kylin.util.ExecAndComp;
-import org.apache.parquet.Strings;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparderEnv;
import org.junit.After;
@@ -68,8 +67,8 @@ public class CalciteDynamicTest extends NLocalWithSparkSessionTest {
List<List<String>> results = transformToString(rows);
for (int i = 0; i < benchmark.size(); i++) {
if (!ListUtils.isEqualList(benchmark.get(i), results.get(i))) {
- String expected = Strings.join(benchmark.get(i), ",");
- String actual1 = Strings.join(results.get(i), ",");
+ String expected = String.join(",", benchmark.get(i));
+ String actual1 = String.join(",", results.get(i));
fail("expected: " + expected + ", results: " + actual1);
}
}
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/NBuildAndQuerySnapshotTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/NBuildAndQuerySnapshotTest.java
index 43867cc5f5..4f3afa6375 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/NBuildAndQuerySnapshotTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/NBuildAndQuerySnapshotTest.java
@@ -42,7 +42,6 @@ import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.util.ExecAndComp;
-import org.apache.parquet.Strings;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparderEnv;
import org.junit.Assert;
@@ -140,8 +139,8 @@ public class NBuildAndQuerySnapshotTest extends NLocalWithSparkSessionTest {
}
private void setPartitions(NSparkSnapshotJob job, Set<String> partitions) {
- job.setParam("partitions", Strings.join(partitions, ","));
- job.getSnapshotBuildingStep().setParam("partitions", Strings.join(partitions, ","));
+ job.setParam("partitions", String.join(",", partitions));
+ job.getSnapshotBuildingStep().setParam("partitions", String.join(",", partitions));
}
}
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/TimeZoneQueryTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/TimeZoneQueryTest.java
index 3f0f5dab4e..26eca957d8 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/TimeZoneQueryTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/TimeZoneQueryTest.java
@@ -46,7 +46,6 @@ import org.apache.kylin.query.engine.PrepareSqlStateParam;
import org.apache.kylin.query.pushdown.SparkSqlClient;
import org.apache.kylin.rest.util.PrepareSQLUtils;
import org.apache.kylin.util.ExecAndComp;
-import org.apache.parquet.Strings;
import org.apache.spark.SparkConf;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparderEnv;
@@ -128,9 +127,9 @@ public class TimeZoneQueryTest extends NLocalWithSparkSessionTest {
for (int i = 0; i < jdbc.size(); i++) {
if (!ListUtils.isEqualList(calciteDf.get(i), pushDown.get(i))
&& !ListUtils.isEqualList(calciteDf.get(i), jdbc.get(i))) {
- String expected = Strings.join(pushDown.get(i), ",");
- String actual = Strings.join(jdbc.get(i), ",");
- String actual2 = Strings.join(calciteDf.get(i), ",");
+ String expected = String.join(",", pushDown.get(i));
+ String actual = String.join(",", jdbc.get(i));
+ String actual2 = String.join(",", calciteDf.get(i));
fail("expected: " + expected + ", actual: " + actual + ", actual2: " + actual2);
}
}
@@ -188,11 +187,11 @@ public class TimeZoneQueryTest extends NLocalWithSparkSessionTest {
&& !ListUtils.isEqualList(benchmark.get(i), setTimestampPushdownResults.get(i))
&& !ListUtils.isEqualList(benchmark.get(i), setStringResults.get(i))
&& !ListUtils.isEqualList(benchmark.get(i), setStringPushdownResults.get(i))) {
- String expected = Strings.join(benchmark.get(i), ",");
- String actual1 = Strings.join(setTimestampResults.get(i), ",");
- String actual2 = Strings.join(setTimestampPushdownResults.get(i), ",");
- String actual3 = Strings.join(setStringResults.get(i), ",");
- String actual4 = Strings.join(setStringPushdownResults.get(i), ",");
+ String expected = String.join(",", benchmark.get(i));
+ String actual1 = String.join(",", setTimestampResults.get(i));
+ String actual2 = String.join(",", setTimestampPushdownResults.get(i));
+ String actual3 = String.join(",", setStringResults.get(i));
+ String actual4 = String.join(",", setStringPushdownResults.get(i));
fail("expected: " + expected + ", setTimestampResults: " + actual1 + ", setTimestampPushdownResults: "
+ actual2 + ", setStringResults: " + actual3 + ", setStringPushdownResults: " + actual4);
}
diff --git a/src/metadata-server/pom.xml b/src/metadata-server/pom.xml
index 5f16c8cc7d..32b8984ea3 100644
--- a/src/metadata-server/pom.xml
+++ b/src/metadata-server/pom.xml
@@ -50,6 +50,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-common-service</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.springfox</groupId>
+ <artifactId>springfox-boot-starter</artifactId>
+ </dependency>
<dependency>
<groupId>org.springframework</groupId>
diff --git a/src/query-server/pom.xml b/src/query-server/pom.xml
index d770732813..3ed9aecaab 100644
--- a/src/query-server/pom.xml
+++ b/src/query-server/pom.xml
@@ -74,6 +74,11 @@
<artifactId>kylin-integration-service</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.springfox</groupId>
+ <artifactId>springfox-boot-starter</artifactId>
+ </dependency>
+
<!--Env & Test-->
<dependency>
<groupId>org.junit.jupiter</groupId>
@@ -222,6 +227,12 @@
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.jayway.jsonpath</groupId>
+ <artifactId>json-path</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
diff --git a/src/query-service/pom.xml b/src/query-service/pom.xml
index d4ab4fe0d8..26fbb531f6 100644
--- a/src/query-service/pom.xml
+++ b/src/query-service/pom.xml
@@ -108,11 +108,6 @@
<artifactId>hive-metastore</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>test</scope>
- </dependency>
<!-- Spring Security -->
<dependency>
<groupId>org.springframework.security</groupId>
@@ -173,11 +168,6 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkContextHealthIndicator.java b/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkContextHealthIndicator.java
index 08cc86c105..267acfca3e 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkContextHealthIndicator.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkContextHealthIndicator.java
@@ -20,10 +20,11 @@ package org.apache.kylin.rest.health;
import org.apache.spark.sql.SparderEnv;
import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.stereotype.Component;
@Component
-public class SparkContextHealthIndicator extends AbstractKylinHealthIndicator {
+public class SparkContextHealthIndicator implements HealthIndicator {
@Override
public Health health() {
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkSqlContextHealthIndicator.java b/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkSqlContextHealthIndicator.java
index ae245dc540..e6ca11cdf2 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkSqlContextHealthIndicator.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/health/SparkSqlContextHealthIndicator.java
@@ -19,10 +19,11 @@ package org.apache.kylin.rest.health;
import org.apache.kylin.rest.monitor.SparkContextCanary;
import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.stereotype.Component;
@Component
-public class SparkSqlContextHealthIndicator extends AbstractKylinHealthIndicator {
+public class SparkSqlContextHealthIndicator implements HealthIndicator {
@Override
public Health health() {
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/AysncQueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/AysncQueryServiceTest.java
index edb498fce2..282824ac65 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/AysncQueryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/AysncQueryServiceTest.java
@@ -64,7 +64,6 @@ import org.apache.kylin.query.exception.NAsyncQueryIllegalParamException;
import org.apache.kylin.query.pushdown.SparkSqlClient;
import org.apache.kylin.query.util.AsyncQueryUtil;
import org.apache.kylin.rest.response.SQLResponse;
-import org.apache.parquet.Strings;
import org.apache.poi.ss.usermodel.CellType;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFRow;
@@ -984,7 +983,7 @@ public class AysncQueryServiceTest extends ServiceTestBase {
try (FSDataOutputStream os = fileSystem
.create(new Path(asyncQueryResultDir, AsyncQueryUtil.getMetaDataFileName())); //
OutputStreamWriter osw = new OutputStreamWriter(os, StandardCharsets.UTF_8)) { //
- String metaString = Strings.join(columnNames, ",") + "\n" + Strings.join(dataTypes, ",");
+ String metaString = String.join(",", columnNames) + "\n" + String.join(",", dataTypes);
osw.write(metaString);
if (needMeta) {
fileSystem.createNewFile(new Path(asyncQueryResultDir, AsyncQueryUtil.getMetaDataFileName()));
diff --git a/src/second-storage/core-ui/pom.xml b/src/second-storage/core-ui/pom.xml
index 7e3f843756..dea9225ee0 100644
--- a/src/second-storage/core-ui/pom.xml
+++ b/src/second-storage/core-ui/pom.xml
@@ -58,9 +58,14 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kap-second-storage-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.springfox</groupId>
+ <artifactId>springfox-boot-starter</artifactId>
+ </dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/src/second-storage/core/pom.xml b/src/second-storage/core/pom.xml
index a1b066c359..f5102b72d9 100644
--- a/src/second-storage/core/pom.xml
+++ b/src/second-storage/core/pom.xml
@@ -51,12 +51,6 @@
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-metadata</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
diff --git a/src/second-storage/core/src/test/java/io/kyligence/kap/secondstorage/metadata/NodeGroupTest.java b/src/second-storage/core/src/test/java/io/kyligence/kap/secondstorage/metadata/NodeGroupTest.java
index 48e939cab6..e9e6db633e 100644
--- a/src/second-storage/core/src/test/java/io/kyligence/kap/secondstorage/metadata/NodeGroupTest.java
+++ b/src/second-storage/core/src/test/java/io/kyligence/kap/secondstorage/metadata/NodeGroupTest.java
@@ -18,7 +18,7 @@
package io.kyligence.kap.secondstorage.metadata;
-import io.kyligence.kap.shaded.curator.org.apache.curator.shaded.com.google.common.collect.Lists;
+import com.google.common.collect.Lists;
import org.junit.Assert;
import org.junit.Test;
diff --git a/src/server/pom.xml b/src/server/pom.xml
index 6c4d716f10..d26f87b24d 100644
--- a/src/server/pom.xml
+++ b/src/server/pom.xml
@@ -139,6 +139,14 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>io.springfox</groupId>
+ <artifactId>springfox-boot-starter</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-config</artifactId>
+ </dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
@@ -181,13 +189,21 @@
<groupId>com.taobao.arthas</groupId>
<artifactId>arthas-spring-boot-starter</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ </dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
</dependency>
<dependency>
- <groupId>org.postgresql</groupId>
- <artifactId>postgresql</artifactId>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
diff --git a/src/systools/src/main/java/org/apache/kylin/rest/config/KylinRequestHandlerSelectors.java b/src/server/src/main/java/org/apache/kylin/rest/config/KylinRequestHandlerSelectors.java
similarity index 100%
rename from src/systools/src/main/java/org/apache/kylin/rest/config/KylinRequestHandlerSelectors.java
rename to src/server/src/main/java/org/apache/kylin/rest/config/KylinRequestHandlerSelectors.java
diff --git a/src/systools/src/main/java/org/apache/kylin/rest/config/SecurityConfig.java b/src/server/src/main/java/org/apache/kylin/rest/config/SecurityConfig.java
similarity index 100%
rename from src/systools/src/main/java/org/apache/kylin/rest/config/SecurityConfig.java
rename to src/server/src/main/java/org/apache/kylin/rest/config/SecurityConfig.java
diff --git a/src/systools/src/main/java/org/apache/kylin/rest/config/SwaggerConfig.java b/src/server/src/main/java/org/apache/kylin/rest/config/SwaggerConfig.java
similarity index 100%
rename from src/systools/src/main/java/org/apache/kylin/rest/config/SwaggerConfig.java
rename to src/server/src/main/java/org/apache/kylin/rest/config/SwaggerConfig.java
diff --git a/src/server/src/main/java/org/apache/kylin/rest/health/FileSystemHealthIndicator.java b/src/server/src/main/java/org/apache/kylin/rest/health/FileSystemHealthIndicator.java
index bba9e10dbf..ed23640a1b 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/health/FileSystemHealthIndicator.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/health/FileSystemHealthIndicator.java
@@ -32,15 +32,15 @@ import org.apache.kylin.rest.config.initialize.AfterMetadataReadyEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import com.google.common.annotations.VisibleForTesting;
@Component
-public class FileSystemHealthIndicator extends AbstractKylinHealthIndicator
- implements ApplicationListener<AfterMetadataReadyEvent> {
- public static final Logger logger = LoggerFactory.getLogger(FileSystemHealthIndicator.class);
+public class FileSystemHealthIndicator implements HealthIndicator, ApplicationListener<AfterMetadataReadyEvent> {
+ private static final Logger logger = LoggerFactory.getLogger(FileSystemHealthIndicator.class);
private static final ScheduledExecutorService FILE_SYSTEM_HEALTH_EXECUTOR = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("FileSystemHealthChecker"));
private volatile boolean isHealth = false;
diff --git a/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java b/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java
index 42a5b29210..ca20cc13a3 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/health/MetaStoreHealthIndicator.java
@@ -26,23 +26,23 @@ import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.StringEntity;
-import org.apache.kylin.common.util.NamedThreadFactory;
-import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.persistence.transaction.UnitOfWorkParams;
+import org.apache.kylin.common.util.NamedThreadFactory;
+import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.rest.config.initialize.AfterMetadataReadyEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import com.google.common.annotations.VisibleForTesting;
@Component
-public class MetaStoreHealthIndicator extends AbstractKylinHealthIndicator
- implements ApplicationListener<AfterMetadataReadyEvent> {
- public static final Logger logger = LoggerFactory.getLogger(MetaStoreHealthIndicator.class);
+public class MetaStoreHealthIndicator implements HealthIndicator, ApplicationListener<AfterMetadataReadyEvent> {
+ private static final Logger logger = LoggerFactory.getLogger(MetaStoreHealthIndicator.class);
private static final String UNIT_NAME = "_health";
private static final String HEALTH_ROOT_PATH = "/" + UNIT_NAME;
@@ -52,10 +52,28 @@ public class MetaStoreHealthIndicator extends AbstractKylinHealthIndicator
new NamedThreadFactory("MetaStoreHealthChecker"));
private volatile boolean isHealth = false;
+ private final int warningResponseMs;
+ private final int errorResponseMs;
+
public MetaStoreHealthIndicator() {
- this.config = KylinConfig.getInstanceFromEnv();
- this.warningResponseMs = KapConfig.wrap(config).getMetaStoreHealthWarningResponseMs();
- this.errorResponseMs = KapConfig.wrap(config).getMetaStoreHealthErrorResponseMs();
+ KapConfig wrappedConfig = KapConfig.wrap(KylinConfig.getInstanceFromEnv());
+ this.warningResponseMs = wrappedConfig.getMetaStoreHealthWarningResponseMs();
+ this.errorResponseMs = wrappedConfig.getMetaStoreHealthErrorResponseMs();
+ }
+
+ private void checkTime(long start, String operation) throws InterruptedException {
+ // in case canary was timeout
+ if (Thread.interrupted())
+ throw new InterruptedException();
+
+ long response = System.currentTimeMillis() - start;
+ logger.trace("{} took {} ms", operation, response);
+
+ if (response > errorResponseMs) {
+ throw new IllegalStateException("check time is time out");
+ } else if (response > warningResponseMs) {
+ logger.warn("found warning, {} took {} ms", operation, response);
+ }
}
@Override
diff --git a/src/systools/src/main/java/org/apache/kylin/rest/security/UnauthorisedEntryPoint.java b/src/server/src/main/java/org/apache/kylin/rest/security/UnauthorisedEntryPoint.java
similarity index 100%
rename from src/systools/src/main/java/org/apache/kylin/rest/security/UnauthorisedEntryPoint.java
rename to src/server/src/main/java/org/apache/kylin/rest/security/UnauthorisedEntryPoint.java
diff --git a/src/source-hive/pom.xml b/src/source-hive/pom.xml
index 69d1bb28d6..41e9f9d1c3 100644
--- a/src/source-hive/pom.xml
+++ b/src/source-hive/pom.xml
@@ -42,10 +42,6 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-metadata</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-job</artifactId>
- </dependency>
<!-- Env & Test -->
<dependency>
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
index 308654a884..5b02c09393 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
@@ -37,7 +37,7 @@ import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.kylin.common.KapConfig;
@@ -367,7 +367,7 @@ public class NSparkExecutable extends AbstractExecutable implements ChainedStage
if (getParent() != null) {
String yarnQueue = getParent().getSparkYarnQueue();
// TODO double check if valid yarn queue
- if (!Strings.isNullOrEmpty(yarnQueue)) {
+ if (!StringUtils.isEmpty(yarnQueue)) {
jobOverrides.put("kylin.engine.spark-conf." + SPARK_YARN_QUEUE, yarnQueue);
}
}
diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkSnapshotJobTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkSnapshotJobTest.java
index f360876711..d7bd1f3f15 100644
--- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkSnapshotJobTest.java
+++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/NSparkSnapshotJobTest.java
@@ -39,7 +39,6 @@ import org.apache.kylin.job.execution.NExecutableManager;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.parquet.Strings;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -213,8 +212,8 @@ public class NSparkSnapshotJobTest extends NLocalWithSparkSessionTest {
}
private void setPartitions(NSparkSnapshotJob job, Set<String> partitions) {
- job.setParam("partitions", Strings.join(partitions, ","));
- job.getSnapshotBuildingStep().setParam("partitions", Strings.join(partitions, ","));
+ job.setParam("partitions", String.join(",", partitions));
+ job.getSnapshotBuildingStep().setParam("partitions", String.join(",", partitions));
}
@Test
diff --git a/src/systools/pom.xml b/src/systools/pom.xml
index ea156c7cb4..e64ff36e53 100644
--- a/src/systools/pom.xml
+++ b/src/systools/pom.xml
@@ -34,36 +34,6 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-metadata</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-datasource-sdk</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-engine-spark</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-swagger</artifactId>
- </dependency>
- <dependency>
- <groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-guava20</artifactId>
- </dependency>
-
- <!-- spring aop -->
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjrt</artifactId>
@@ -72,144 +42,50 @@
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
</dependency>
-
- <!-- spring -->
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-aop</artifactId>
- </dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context-support</artifactId>
</dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-web</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.session</groupId>
- <artifactId>spring-session-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-autoconfigure</artifactId>
- <exclusions>
- <exclusion>
- <artifactId>spring-context</artifactId>
- <groupId>org.springframework</groupId>
- </exclusion>
- <exclusion>
- <artifactId>spring-core</artifactId>
- <groupId>org.springframework</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-actuator</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-logging</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>spring-context</artifactId>
- <groupId>org.springframework</groupId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
</dependency>
-
- <!-- Spring Security -->
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-ldap</artifactId>
</dependency>
- <dependency>
- <groupId>org.springframework.security.extensions</groupId>
- <artifactId>spring-security-saml2-core</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-ext-jdk15on</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-test</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-web</artifactId>
- </dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-acl</artifactId>
</dependency>
- <!-- hadoop -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
</dependency>
-
- <!-- Swagger -->
- <dependency>
- <groupId>io.springfox</groupId>
- <artifactId>springfox-boot-starter</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.mapstruct</groupId>
- <artifactId>mapstruct</artifactId>
- </dependency>
-
- <!-- util -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>redis.clients</groupId>
- <artifactId>jedis</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<scope>provided</scope>
</dependency>
+
<dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- <scope>compile</scope>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-test</artifactId>
+ <scope>test</scope>
</dependency>
-
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
@@ -240,5 +116,12 @@
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
+
+ <!--FIX ME-->
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-engine-spark</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/src/systools/src/main/java/org/apache/kylin/rest/health/AbstractKylinHealthIndicator.java b/src/systools/src/main/java/org/apache/kylin/rest/health/AbstractKylinHealthIndicator.java
deleted file mode 100644
index f2e24f38d0..0000000000
--- a/src/systools/src/main/java/org/apache/kylin/rest/health/AbstractKylinHealthIndicator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.rest.health;
-
-import org.apache.kylin.common.KylinConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.boot.actuate.health.HealthIndicator;
-
-public abstract class AbstractKylinHealthIndicator implements HealthIndicator {
- public static final Logger logger = LoggerFactory.getLogger(AbstractKylinHealthIndicator.class);
-
- protected KylinConfig config;
-
- protected int warningResponseMs;
- protected int errorResponseMs;
-
- protected void checkTime(long start, String operation) throws InterruptedException {
- // in case canary was timeout
- if (Thread.interrupted())
- throw new InterruptedException();
-
- long response = System.currentTimeMillis() - start;
- logger.trace("{} took {} ms", operation, response);
-
- if (response > errorResponseMs) {
- throw new RuntimeException("check time is time out");
- } else if (response > warningResponseMs) {
- logger.warn("found warning, {} took {} ms", operation, response);
- }
- }
-}
diff --git a/src/systools/src/main/java/org/apache/kylin/rest/security/KerberosLoginManager.java b/src/systools/src/main/java/org/apache/kylin/rest/security/KerberosLoginManager.java
index f20e658b88..6cfe74dbda 100644
--- a/src/systools/src/main/java/org/apache/kylin/rest/security/KerberosLoginManager.java
+++ b/src/systools/src/main/java/org/apache/kylin/rest/security/KerberosLoginManager.java
@@ -25,6 +25,7 @@ import java.security.PrivilegedAction;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.kylin.common.KapConfig;
@@ -117,7 +118,8 @@ public class KerberosLoginManager {
}
}
- private boolean checkExistsTablesAccess(UserGroupInformation ugi, String project) {
+ @VisibleForTesting
+ boolean checkExistsTablesAccess(UserGroupInformation ugi, String project) {
val kapConfig = KapConfig.getInstanceFromEnv();
val projectManager = NProjectManager.getInstance(kapConfig.getKylinConfig());
return ugi.doAs((PrivilegedAction<Boolean>) () -> {
diff --git a/src/systools/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java b/src/systools/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java
index 6a70456aba..c948ab3e53 100644
--- a/src/systools/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java
+++ b/src/systools/src/main/java/org/apache/kylin/rest/util/AclPermissionUtil.java
@@ -27,7 +27,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.exception.KylinException;
diff --git a/src/systools/src/test/java/org/apache/kylin/rest/security/KerberosLoginManagerTest.java b/src/systools/src/test/java/org/apache/kylin/rest/security/KerberosLoginManagerTest.java
index 0405f03871..e5422160c2 100644
--- a/src/systools/src/test/java/org/apache/kylin/rest/security/KerberosLoginManagerTest.java
+++ b/src/systools/src/test/java/org/apache/kylin/rest/security/KerberosLoginManagerTest.java
@@ -18,15 +18,12 @@
package org.apache.kylin.rest.security;
-import java.lang.reflect.Method;
-
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.springframework.util.ReflectionUtils;
public class KerberosLoginManagerTest extends NLocalFileMetadataTestCase {
@Before
@@ -41,11 +38,8 @@ public class KerberosLoginManagerTest extends NLocalFileMetadataTestCase {
@Test
public void testCheckExistsTablesAccess() throws Exception {
- Method method = KerberosLoginManager.class.getDeclaredMethod("checkExistsTablesAccess",
- new Class[] { UserGroupInformation.class, String.class });
- method.setAccessible(true);
UserGroupInformation userGroupInformation = UserGroupInformation.getCurrentUser();
- Assert.assertTrue((boolean) ReflectionUtils.invokeMethod(method, KerberosLoginManager.getInstance(),
- userGroupInformation, "ssb"));
+ KerberosLoginManager kerberosLoginManager = KerberosLoginManager.getInstance();
+ Assert.assertTrue(kerberosLoginManager.checkExistsTablesAccess(userGroupInformation, "ssb"));
}
}
diff --git a/src/systools/src/test/java/org/apache/kylin/rest/security/UserAclManagerTest.java b/src/systools/src/test/java/org/apache/kylin/rest/security/UserAclManagerTest.java
index 951dad6970..67000f1e1e 100644
--- a/src/systools/src/test/java/org/apache/kylin/rest/security/UserAclManagerTest.java
+++ b/src/systools/src/test/java/org/apache/kylin/rest/security/UserAclManagerTest.java
@@ -18,7 +18,7 @@
package org.apache.kylin.rest.security;
-import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.junit.After;
import org.junit.Assert;
diff --git a/src/systools/src/test/resources/ehcache.xml b/src/systools/src/test/resources/ehcache.xml
index 7f997b56df..e69de29bb2 100644
--- a/src/systools/src/test/resources/ehcache.xml
+++ b/src/systools/src/test/resources/ehcache.xml
@@ -1,19 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<ehcache name="DefaultCacheName" maxBytesLocalHeap="1M">
-</ehcache>
diff --git a/src/tool/pom.xml b/src/tool/pom.xml
index d5c563a0eb..24dbd4ce9c 100644
--- a/src/tool/pom.xml
+++ b/src/tool/pom.xml
@@ -42,38 +42,11 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-streaming</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-dbcp2</artifactId>
- </dependency>
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-core</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-xml</artifactId>
- </dependency>
-
- <!-- Spark dependency -->
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
- <version>${spark.version}</version>
- <scope>provided</scope>
- </dependency>
-
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
@@ -97,6 +70,13 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
+
+ <!-- Spark dependency -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.12</artifactId>
+ <scope>provided</scope>
+ </dependency>
<!-- mysql -->
<dependency>
<groupId>mysql</groupId>
@@ -118,7 +98,16 @@
</exclusions>
<scope>provided</scope>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-jdbc</artifactId>
@@ -131,17 +120,13 @@
</exclusions>
</dependency>
+ <!-- Test -->
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- <version>2.10.2</version>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
-
- <!-- Test -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
@@ -160,7 +145,6 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
- <version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -181,7 +165,6 @@
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
- <version>${awaitility.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -194,11 +177,6 @@
<artifactId>powermock-api-mockito2</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>com.github.stefanbirkner</groupId>
<artifactId>system-lambda</artifactId>
diff --git a/src/tool/src/main/java/org/apache/kylin/tool/RecCandidateTool.java b/src/tool/src/main/java/org/apache/kylin/tool/RecCandidateTool.java
index 953b399cb4..0e553458ad 100644
--- a/src/tool/src/main/java/org/apache/kylin/tool/RecCandidateTool.java
+++ b/src/tool/src/main/java/org/apache/kylin/tool/RecCandidateTool.java
@@ -251,7 +251,7 @@ public class RecCandidateTool extends ExecutableApplication {
}
}
- private String getProjectByModelId(String modelId) {
+ String getProjectByModelId(String modelId) {
if (StringUtils.isEmpty(modelId)) {
throw new KylinException(PARAMETER_EMPTY, "model");
}
diff --git a/src/tool/src/test/java/org/apache/kylin/tool/RecCandidateToolTest.java b/src/tool/src/test/java/org/apache/kylin/tool/RecCandidateToolTest.java
new file mode 100644
index 0000000000..e69de29bb2