You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/01/12 09:28:12 UTC
[kylin] 16/17: KYLIN-5399 refactor core-common
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1a00e9ac57de03d3436e153dbc535a5ee5809cd2
Author: qianhao.zhou <z....@gmail.com>
AuthorDate: Sat Nov 12 20:41:02 2022 +0800
KYLIN-5399 refactor core-common
* fix async query cannot run
* fix newten-job.jar cannot run
* add missing dependency
Co-authored-by: qhzhou <qi...@kyligence.io>
---
pom.xml | 18 ++
src/assembly/pom.xml | 4 +
src/common-service/pom.xml | 4 +
.../apache/kylin/rest/config/MetricsConfig.java | 4 +-
.../kylin/rest/controller/NBasicController.java | 17 +-
.../interceptor/ReloadAuthoritiesInterceptor.java | 5 +-
.../rest/security/NUnauthorisedEntryPoint.java | 3 +-
.../org/apache/kylin/rest/util/SparkUIUtil.java | 3 +-
src/core-common/pom.xml | 222 ++++-----------------
.../org/apache/kylin/common/KylinConfigBase.java | 4 +-
.../AsyncProfilerConstants.java} | 23 +--
.../common/persistence/metadata/jdbc/JdbcUtil.java | 2 +-
.../common/persistence/transaction/UnitOfWork.java | 22 +-
.../java/org/apache/kylin/common/util/Unsafe.java | 8 +-
src/core-job/pom.xml | 16 ++
.../kylin/common/util/SystemInfoCollector.java | 0
.../apache/kylin/metrics/HdfsCapacityMetrics.java | 0
.../kylin/metrics/HdfsCapacityMetricsTest.java | 0
src/core-metadata/pom.xml | 26 +--
.../kylin/metadata/model/ComputedColumnDesc.java | 9 -
.../apache/kylin/metadata/model/FunctionDesc.java | 5 +-
.../metadata/model/schema/ModelEdgeCollector.java | 12 +-
.../org/apache/kylin/source/jdbc/H2Database.java | 2 +-
src/{core-job => core-metrics}/pom.xml | 128 ++++++------
.../kylin/common/metrics/MetricsCategory.java | 0
.../apache/kylin/common/metrics/MetricsConfig.java | 0
.../kylin/common/metrics/MetricsController.java | 0
.../apache/kylin/common/metrics/MetricsGroup.java | 0
.../common/metrics/MetricsInfluxdbReporter.java | 0
.../apache/kylin/common/metrics/MetricsName.java | 0
.../apache/kylin/common/metrics/MetricsObject.java | 0
.../kylin/common/metrics/MetricsObjectType.java | 0
.../kylin/common/metrics/MetricsReporter.java | 0
.../apache/kylin/common/metrics/MetricsTag.java | 0
.../common/metrics/context/ClusterContext.java | 0
.../kylin/common/metrics/context/ClusterInfo.java | 0
.../common/metrics/gauges/QueryRatioGauge.java | 0
.../metrics/prometheus/PrometheusMetrics.java | 0
.../common/metrics/reporter/InfluxdbReporter.java | 2 +-
.../metrics/reporter/ServerModeMetricFilter.java | 0
.../common/metrics/service/InfluxDBInstance.java | 4 +-
.../metrics/service/JobStatusMonitorMetric.java | 0
.../kylin/common/metrics/service/MonitorDao.java | 0
.../common/metrics/service/MonitorMetric.java | 0
.../metrics/service/MonitorMetricOperation.java | 0
.../common/metrics/service/QueryMonitorMetric.java | 0
.../apache/kylin/common/util/InfluxDBUtils.java | 0
.../kylin/common/metric/InfluxDBInstanceTest.java | 0
.../kylin/common/metric/InfluxMetricsTest.java | 0
.../kylin/common/metric/MetricsControllerTest.java | 16 +-
.../apache/kylin/common/metric/MonitorDaoTest.java | 0
.../kylin/rest/controller/BaseController.java | 19 +-
.../kylin/rest/filter/SegmentsRequestFilter.java | 3 +-
src/datasource-sdk/pom.xml | 27 +--
src/datasource-service/pom.xml | 4 +
src/{core-job => distributed-lock-ext}/pom.xml | 107 +++++-----
.../lock/curator/CuratorDistributedLock.java | 0
.../curator/CuratorDistributedLockFactory.java | 0
.../lock/jdbc/JdbcDistributedLockFactory.java | 0
.../common/lock/jdbc/JdbcDistributedLockUtil.java | 24 ++-
.../java/org/apache/kylin/common/util/ZKUtil.java | 8 +-
.../common/lock/DistributedLockFactoryTest.java | 11 +-
.../curator/CuratorDistributedLockFactoryTest.java | 43 ++--
.../lock/jdbc/JdbcDistributedLockFactoryTest.java | 13 +-
src/kylin-it/pom.xml | 5 +
.../kylin/streaming/StreamingMergeEntryTest.java | 9 +
.../kylin/common/util/AddTableNameSqlVisitor.java | 0
.../common/util/ModifyTableNameSqlVisitor.java | 0
.../kylin/rest/service/ModelSemanticHelper.java | 9 +-
src/query/pom.xml | 4 +
src/server/pom.xml | 24 +++
.../org/apache/kylin/rest/QueryNodeFilter.java | 7 +-
src/spark-project/engine-build-sdk/pom.xml | 1 -
src/spark-project/engine-spark/pom.xml | 4 +
.../kylin/engine/spark/job/JobManagerTest.java | 29 +--
src/spark-project/spark-common/pom.xml | 5 +
.../kylin/common/asyncprofiler/AsyncArchUtil.java | 0
.../kylin/common/asyncprofiler/AsyncProfiler.java | 12 --
.../common/asyncprofiler/AsyncProfilerUtils.java | 0
.../org/apache/kylin/source/dfs/DFSFileTable.java | 0
.../kylin/source/dfs/DFSFileTableReader.java | 4 +-
.../libasyncProfiler-linux-arm64.so | Bin 0 -> 298256 bytes
.../libasyncProfiler-linux-x64.so | Bin 0 -> 303903 bytes
.../async-profiler-lib/libasyncProfiler-mac.so | Bin 0 -> 634272 bytes
.../async-profiler-lib/linux64/libasyncProfiler.so | Bin 0 -> 314098 bytes
.../async-profiler-lib/macOS/libasyncProfiler.so | Bin 0 -> 239460 bytes
.../AsyncProfilerExecutorPlugin.scala | 0
.../common/asyncprofiler/AsyncProfilerTool.scala | 2 +-
.../kylin/common/asyncprofiler/Message.scala | 4 +-
.../common/asyncprofiler/AsyncArchUtilTest.java | 0
.../common/asyncprofiler/AsyncProfilerTest.java | 18 --
.../asyncprofiler/AsyncProfilerToolTest.java | 0
.../asyncprofiler/AsyncProfilerUtilsTest.java | 6 +-
.../kylin/common/asyncprofiler/MessageTest.java | 3 +-
src/systools/pom.xml | 22 +-
95 files changed, 425 insertions(+), 561 deletions(-)
diff --git a/pom.xml b/pom.xml
index ad78eecd6d..40eb6b25f9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -317,6 +317,8 @@
<module>src/query</module>
<module>src/core-common</module>
<module>src/core-metadata</module>
+ <module>src/distributed-lock-ext</module>
+ <module>src/core-metrics</module>
<module>src/source-hive</module>
<module>src/server</module>
<module>src/spark-project/kylin-soft-affinity-cache</module>
@@ -523,6 +525,22 @@
<artifactId>kylin-integration-service</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>distributed-lock-ext</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-metrics</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-streaming</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
<!-- arthas -->
<dependency>
diff --git a/src/assembly/pom.xml b/src/assembly/pom.xml
index 2a56110bfc..5436fda9bd 100644
--- a/src/assembly/pom.xml
+++ b/src/assembly/pom.xml
@@ -88,6 +88,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-query</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ </dependency>
<!-- Env -->
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/src/common-service/pom.xml b/src/common-service/pom.xml
index 6226d6ea78..d6fcd55cc1 100644
--- a/src/common-service/pom.xml
+++ b/src/common-service/pom.xml
@@ -59,6 +59,10 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-fileupload</groupId>
+ <artifactId>commons-fileupload</artifactId>
+ </dependency>
<dependency>
<groupId>net.sf.ehcache</groupId>
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java
index a0126eee3e..0289709b62 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java
@@ -28,11 +28,11 @@ import java.util.stream.Collectors;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.NamedThreadFactory;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.common.metrics.MetricsController;
import org.apache.kylin.common.metrics.MetricsGroup;
+import org.apache.kylin.common.util.NamedThreadFactory;
import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.cluster.ClusterManager;
import org.apache.kylin.rest.config.initialize.MetricsRegistry;
import org.springframework.beans.factory.annotation.Autowired;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
index 4ffe05c013..884a807128 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
@@ -83,7 +83,6 @@ import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.execution.JobTypeEnum;
@@ -188,7 +187,7 @@ public class NBasicController {
if (kylinException != null) {
cause = kylinException;
}
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause);
+ return new ErrorResponse(req.getRequestURL().toString(), cause);
}
@ResponseStatus(HttpStatus.FORBIDDEN)
@@ -196,7 +195,7 @@ public class NBasicController {
@ResponseBody
ErrorResponse handleForbidden(HttpServletRequest req, Exception ex) {
getLogger().error("", ex);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
@ResponseStatus(HttpStatus.NOT_FOUND)
@@ -204,7 +203,7 @@ public class NBasicController {
@ResponseBody
ErrorResponse handleNotFound(HttpServletRequest req, Exception ex) {
getLogger().error("", ex);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -228,7 +227,7 @@ public class NBasicController {
ErrorResponse handleAccessDenied(HttpServletRequest req, Throwable ex) {
getLogger().error("", ex);
KylinException e = new KylinException(ACCESS_DENIED, MsgPicker.getMsg().getAccessDeny());
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e);
+ return new ErrorResponse(req.getRequestURL().toString(), e);
}
@ResponseStatus(HttpStatus.BAD_REQUEST)
@@ -238,7 +237,7 @@ public class NBasicController {
ErrorResponse handleInvalidRequestParam(HttpServletRequest req, Throwable ex) {
KylinException e = new KylinException(INVALID_PARAMETER, ex);
getLogger().error("", e);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e);
+ return new ErrorResponse(req.getRequestURL().toString(), e);
}
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -247,14 +246,14 @@ public class NBasicController {
ErrorResponse handleErrorCode(HttpServletRequest req, Throwable ex) {
getLogger().error("", ex);
KylinException cause = (KylinException) ex;
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause);
+ return new ErrorResponse(req.getRequestURL().toString(), cause);
}
@ResponseStatus(HttpStatus.BAD_REQUEST)
@ExceptionHandler(MethodArgumentNotValidException.class)
@ResponseBody
ErrorResponse handleInvalidArgument(HttpServletRequest request, MethodArgumentNotValidException ex) {
- val response = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request), ex);
+ val response = new ErrorResponse(request.getRequestURL().toString(), ex);
val target = ex.getBindingResult().getTarget();
if (target instanceof Validation) {
response.setMsg(((Validation) target).getErrorMessage(ex.getBindingResult().getFieldErrors()));
@@ -272,7 +271,7 @@ public class NBasicController {
ErrorResponse handleUnauthorized(HttpServletRequest req, Throwable ex) {
KylinException e = new KylinException(USER_UNAUTHORIZED, ex);
getLogger().error("", e);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
protected void checkRequiredArg(String fieldName, Object fieldValue) {
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java
index 0d8714647a..1d48d51d66 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java
@@ -31,10 +31,9 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.rest.response.ErrorResponse;
import org.apache.kylin.rest.service.UserService;
-import org.apache.kylin.common.util.Unsafe;
-import org.apache.kylin.metadata.user.ManagedUser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.annotation.Order;
@@ -84,7 +83,7 @@ public class ReloadAuthoritiesInterceptor extends HandlerInterceptorAdapter {
if (present) {
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
response.setContentType(MediaType.APPLICATION_JSON_VALUE);
- ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request),
+ ErrorResponse errorResponse = new ErrorResponse(request.getRequestURL().toString(),
new KylinException(USER_DATA_SOURCE_CONNECTION_FAILED,
MsgPicker.getMsg().getlDapUserDataSourceConnectionFailed()));
response.setCharacterEncoding("UTF-8");
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java b/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java
index 6c376b81f7..3aceb2ad5c 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java
@@ -37,7 +37,6 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.rest.response.ErrorResponse;
-import org.apache.kylin.common.util.Unsafe;
import org.springframework.http.MediaType;
import org.springframework.ldap.CommunicationException;
import org.springframework.security.authentication.DisabledException;
@@ -94,7 +93,7 @@ public class NUnauthorisedEntryPoint implements AuthenticationEntryPoint {
throws IOException {
response.setStatus(statusCode);
response.setContentType(MediaType.APPLICATION_JSON_VALUE);
- ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request), ex);
+ ErrorResponse errorResponse = new ErrorResponse(request.getRequestURL().toString(), ex);
String errorStr = JsonUtil.writeValueAsIndentString(errorResponse);
response.setCharacterEncoding("UTF-8");
PrintWriter writer = response.getWriter();
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java b/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java
index a814b725d0..51a01d6c1e 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java
@@ -28,7 +28,6 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.kylin.common.util.Unsafe;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpHeaders;
@@ -59,7 +58,7 @@ public class SparkUIUtil {
final HttpMethod method = HttpMethod.resolve(servletRequest.getMethod());
try (ClientHttpResponse response = execute(target, method, proxyLocationBase)) {
- rewrite(response, servletResponse, method, Unsafe.getUrlFromHttpServletRequest(servletRequest),
+ rewrite(response, servletResponse, method, servletRequest.getRequestURL().toString(),
REDIRECT_THRESHOLD, proxyLocationBase);
}
}
diff --git a/src/core-common/pom.xml b/src/core-common/pom.xml
index 6c1211efe8..47bd12746c 100644
--- a/src/core-common/pom.xml
+++ b/src/core-common/pom.xml
@@ -37,9 +37,8 @@
<dependencies>
<dependency>
<groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-curator</artifactId>
+ <artifactId>kap-external-guava20</artifactId>
</dependency>
-
<dependency>
<groupId>io.kyligence</groupId>
<artifactId>kyligence-config-external</artifactId>
@@ -50,67 +49,25 @@
</exclusion>
</exclusions>
</dependency>
-
- <!-- provided -->
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.tomcat.embed</groupId>
- <artifactId>tomcat-embed-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-email</artifactId>
</dependency>
- <dependency>
- <groupId>commons-fileupload</groupId>
- <artifactId>commons-fileupload</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
- <dependency>
- <groupId>com.lmax</groupId>
- <artifactId>disruptor</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
+ <artifactId>log4j-web</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-web</artifactId>
+ <artifactId>log4j-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
@@ -121,69 +78,42 @@
<artifactId>log4j-slf4j-impl</artifactId>
</dependency>
<dependency>
- <groupId>org.mybatis</groupId>
- <artifactId>mybatis</artifactId>
+ <groupId>javax.validation</groupId>
+ <artifactId>validation-api</artifactId>
</dependency>
<dependency>
- <groupId>org.mybatis.dynamic-sql</groupId>
- <artifactId>mybatis-dynamic-sql</artifactId>
- </dependency>
-
- <!-- dropwizard -->
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </dependency>
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-jvm</artifactId>
- </dependency>
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-json</artifactId>
- </dependency>
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-jmx</artifactId>
- </dependency>
-
- <!-- micrometer -->
- <dependency>
- <groupId>io.micrometer</groupId>
- <artifactId>micrometer-registry-prometheus</artifactId>
+ <groupId>com.alibaba</groupId>
+ <artifactId>transmittable-thread-local</artifactId>
</dependency>
-
<dependency>
- <groupId>org.latencyutils</groupId>
- <artifactId>LatencyUtils</artifactId>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-jdbc</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-jcl</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
-
+ <!-- provided -->
<dependency>
- <groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-guava20</artifactId>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-core</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
- <groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-influxdb</artifactId>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-dbcp2</artifactId>
+ <scope>provided</scope>
</dependency>
+ <!-- test -->
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<scope>test</scope>
<version>2.10.2</version>
</dependency>
-
- <dependency>
- <groupId>com.github.oshi</groupId>
- <artifactId>oshi-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.javassist</groupId>
- <artifactId>javassist</artifactId>
- </dependency>
- <!--Env & Test-->
<dependency>
<groupId>com.github.sakserv</groupId>
<artifactId>hadoop-mini-clusters-hdfs</artifactId>
@@ -191,25 +121,8 @@
<version>0.1.16</version>
</dependency>
<dependency>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.postgresql</groupId>
- <artifactId>postgresql</artifactId>
- </dependency>
-
- <!-- postgresql 42.2.25 not dependency jcl-over-slf4j, ut testFindContainingJar will fail -->
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-test</artifactId>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
@@ -217,43 +130,6 @@
<artifactId>spring-test</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.12</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <!-- jdbc persistence -->
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-jdbc</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-context-support</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-dbcp2</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.assertj</groupId>
- <artifactId>assertj-core</artifactId>
- </dependency>
-
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
@@ -274,11 +150,6 @@
<artifactId>junit-pioneer</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
@@ -289,7 +160,6 @@
<artifactId>log4j-core</artifactId>
</exclusion>
</exclusions>
- <version>${mockito.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
@@ -301,7 +171,6 @@
<artifactId>log4j-core</artifactId>
</exclusion>
</exclusions>
- <version>${mockito.version}</version>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
@@ -314,32 +183,15 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.alibaba</groupId>
- <artifactId>transmittable-thread-local</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.cloud</groupId>
- <artifactId>spring-cloud-commons</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.integration</groupId>
- <artifactId>spring-integration-jdbc</artifactId>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <scope>test</scope>
</dependency>
-
- <!-- Spark -->
<dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <scope>test</scope>
</dependency>
-
</dependencies>
<build>
<plugins>
@@ -374,6 +226,18 @@
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.0.2</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 557901c837..7bd4da912a 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -19,8 +19,8 @@
package org.apache.kylin.common;
import static java.lang.Math.toIntExact;
-import static org.apache.kylin.common.asyncprofiler.AsyncProfiler.ASYNC_PROFILER_LIB_LINUX_ARM64;
-import static org.apache.kylin.common.asyncprofiler.AsyncProfiler.ASYNC_PROFILER_LIB_LINUX_X64;
+import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_LINUX_ARM64;
+import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_LINUX_X64;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_CONNECTION_URL_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_DRIVER_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_PASS_KEY;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java b/src/core-common/src/main/java/org/apache/kylin/common/constant/AsyncProfilerConstants.java
similarity index 66%
copy from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java
copy to src/core-common/src/main/java/org/apache/kylin/common/constant/AsyncProfilerConstants.java
index 1502b72de5..1949f0db98 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/constant/AsyncProfilerConstants.java
@@ -15,25 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.constant;
-package org.apache.kylin.common.metrics;
+public final class AsyncProfilerConstants {
-public enum MetricsObjectType {
+ private AsyncProfilerConstants() {}
- MAX("max"),
- MIN("min"),
- COUNT("count"),
- FIRST("first"),
- LAST("last");
-
- private String type;
-
- MetricsObjectType(String type) {
- this.type = type;
- }
-
- public String getVal() {
- return this.type;
- }
+ public static final String ASYNC_PROFILER_LIB_MAC = "libasyncProfiler-mac.so";
+ public static final String ASYNC_PROFILER_LIB_LINUX_X64 = "libasyncProfiler-linux-x64.so";
+ public static final String ASYNC_PROFILER_LIB_LINUX_ARM64 = "libasyncProfiler-linux-arm64.so";
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
index 94734b415e..caf500bc14 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
@@ -36,7 +36,6 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.metadata.PersistException;
import org.apache.kylin.common.util.EncryptUtil;
-import org.msgpack.core.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.dao.DataIntegrityViolationException;
@@ -45,6 +44,7 @@ import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.DefaultTransactionDefinition;
+import io.kyligence.kap.guava20.shaded.common.annotations.VisibleForTesting;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
index 280e1987d7..4e576d4e4f 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
@@ -18,7 +18,6 @@
package org.apache.kylin.common.persistence.transaction;
import java.util.List;
-import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@@ -31,11 +30,6 @@ import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ThreadViewResourceStore;
import org.apache.kylin.common.persistence.TombRawResource;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.RandomUtil;
-import org.apache.kylin.common.metrics.MetricsCategory;
-import org.apache.kylin.common.metrics.MetricsGroup;
-import org.apache.kylin.common.metrics.MetricsName;
import org.apache.kylin.common.persistence.UnitMessages;
import org.apache.kylin.common.persistence.event.EndUnit;
import org.apache.kylin.common.persistence.event.Event;
@@ -44,6 +38,8 @@ import org.apache.kylin.common.persistence.event.ResourceDeleteEvent;
import org.apache.kylin.common.persistence.event.ResourceRelatedEvent;
import org.apache.kylin.common.persistence.event.StartUnit;
import org.apache.kylin.common.scheduler.EventBusFactory;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.Unsafe;
import com.google.common.base.Preconditions;
@@ -100,17 +96,6 @@ public class UnitOfWork {
int retry = 0;
val traceId = RandomUtil.randomUUIDStr();
while (retry++ < maxRetry) {
- if (retry > 1) {
- Map<String, String> tags = MetricsGroup.getHostTagMap(params.getUnitName());
-
- if (!GLOBAL_UNIT.equals(params.getUnitName())) {
- MetricsGroup.counterInc(MetricsName.TRANSACTION_RETRY_COUNTER, MetricsCategory.PROJECT,
- params.getUnitName(), tags);
- } else {
- MetricsGroup.counterInc(MetricsName.TRANSACTION_RETRY_COUNTER, MetricsCategory.GLOBAL, "global",
- tags);
- }
- }
val ret = doTransaction(params, retry, traceId);
if (ret.getSecond()) {
@@ -147,9 +132,6 @@ public class UnitOfWork {
long duration = System.currentTimeMillis() - startTransactionTime;
logIfLongTransaction(duration, traceId);
- MetricsGroup.hostTagHistogramUpdate(MetricsName.TRANSACTION_LATENCY, MetricsCategory.PROJECT,
- !GLOBAL_UNIT.equals(params.getUnitName()) ? params.getUnitName() : "global", duration);
-
result = Pair.newPair(ret, true);
} catch (Throwable throwable) {
handleError(throwable, params, retry, traceId);
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java b/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java
index 552b9edc44..cfd276f15d 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java
@@ -24,12 +24,10 @@ import java.text.MessageFormat;
import java.util.Locale;
import java.util.Map;
-import javax.servlet.http.HttpServletRequest;
-
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.SystemPropertiesCache;
import lombok.extern.slf4j.Slf4j;
-import org.apache.kylin.common.SystemPropertiesCache;
/**
* Contains methods that call JDK methods that the
@@ -81,10 +79,6 @@ public class Unsafe {
return temp.format(arguments);
}
- public static String getUrlFromHttpServletRequest(HttpServletRequest request) {
- return request.getRequestURL().toString();
- }
-
/** Reflection usage to work around access flags fails with SecurityManagers
* and likely will not work anymore on runtime classes in Java 9 */
public static void changeAccessibleObject(AccessibleObject accessibleObject, boolean value) {
diff --git a/src/core-job/pom.xml b/src/core-job/pom.xml
index bbdbb0cf2f..a5ac6dff24 100644
--- a/src/core-job/pom.xml
+++ b/src/core-job/pom.xml
@@ -45,6 +45,18 @@
<groupId>io.kyligence.ke</groupId>
<artifactId>kap-external-curator-test</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.kyligence.ke</groupId>
+ <artifactId>kap-external-curator</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-metrics</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-metrics</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
@@ -60,6 +72,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.github.oshi</groupId>
+ <artifactId>oshi-core</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java b/src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java
rename to src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java b/src/core-job/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java
similarity index 100%
rename from src/core-metadata/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java
rename to src/core-job/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java b/src/core-job/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
similarity index 100%
rename from src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
rename to src/core-job/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
diff --git a/src/core-metadata/pom.xml b/src/core-metadata/pom.xml
index 96760a701c..d61c5ab80e 100644
--- a/src/core-metadata/pom.xml
+++ b/src/core-metadata/pom.xml
@@ -36,22 +36,17 @@
</properties>
<dependencies>
-
- <!-- provided -->
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
</dependency>
-
<dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-jdbc</artifactId>
+ <groupId>io.kyligence.ke</groupId>
+ <artifactId>kap-external-influxdb</artifactId>
</dependency>
<dependency>
<groupId>com.tdunning</groupId>
@@ -65,17 +60,13 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
</dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
@@ -109,6 +100,11 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index 5cceedd5df..f807dea94d 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -32,7 +32,6 @@ import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.dialect.HiveSqlDialect;
import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.calcite.sql.util.SqlVisitor;
import org.apache.commons.lang.StringUtils;
@@ -40,7 +39,6 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.common.util.ModifyTableNameSqlVisitor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -211,11 +209,4 @@ public class ComputedColumnDesc implements Serializable {
return innerExpression;
}
- public void changeTableAlias(String oldAlias, String newAlias) {
- SqlVisitor<Object> modifyAlias = new ModifyTableNameSqlVisitor(oldAlias, newAlias);
- SqlNode sqlNode = CalciteParser.getExpNode(getExpression());
- sqlNode.accept(modifyAlias);
- setExpression(sqlNode.toSqlString(HiveSqlDialect.DEFAULT).toString());
- }
-
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index b2a212f84f..28e3603d28 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -42,8 +42,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.directory.api.util.Strings;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureTypeFactory;
@@ -104,7 +103,7 @@ public class FunctionDesc implements Serializable {
break;
}
case FunctionDesc.FUNC_SUM_LC: {
- Preconditions.checkArgument(Strings.isNotEmpty(colDataType),
+ Preconditions.checkArgument(StringUtils.isNotEmpty(colDataType),
"SUM_LC Measure's input type shouldn't be null or empty");
checkSumLCDataType(colDataType);
break;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java
index 8b94d8c7a9..ccda8a9527 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java
@@ -17,6 +17,7 @@
*/
package org.apache.kylin.metadata.model.schema;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -26,14 +27,14 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.model.JoinTableDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.cube.cuboid.NAggregationGroup;
import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import com.google.common.collect.ImmutableBiMap;
@@ -41,7 +42,6 @@ import com.google.common.collect.Maps;
import io.kyligence.kap.guava20.shaded.common.graph.Graph;
import io.kyligence.kap.guava20.shaded.common.graph.MutableGraph;
-import io.kyligence.kap.shaded.curator.org.apache.curator.shaded.com.google.common.collect.Lists;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.val;
@@ -79,9 +79,9 @@ class ModelEdgeCollector {
collectModelSignificant();
collectDimensionAndMeasure();
- collectIndex(indexPlan.getWhitelistLayouts(), SchemaNodeType.WHITE_LIST_INDEX, Lists.newArrayList());
+ collectIndex(indexPlan.getWhitelistLayouts(), SchemaNodeType.WHITE_LIST_INDEX, new ArrayList<>());
collectIndex(indexPlan.getToBeDeletedIndexes().stream().flatMap(index -> index.getLayouts().stream())
- .collect(Collectors.toList()), SchemaNodeType.TO_BE_DELETED_INDEX, Lists.newArrayList());
+ .collect(Collectors.toList()), SchemaNodeType.TO_BE_DELETED_INDEX, new ArrayList<>());
collectIndex(indexPlan.getRuleBaseLayouts(), SchemaNodeType.RULE_BASED_INDEX, indexPlan.getAggShardByColumns());
collectAggGroup();
diff --git a/src/source-hive/src/main/java/org/apache/kylin/source/jdbc/H2Database.java b/src/core-metadata/src/main/java/org/apache/kylin/source/jdbc/H2Database.java
similarity index 100%
rename from src/source-hive/src/main/java/org/apache/kylin/source/jdbc/H2Database.java
rename to src/core-metadata/src/main/java/org/apache/kylin/source/jdbc/H2Database.java
index 2df4a82cda..15e933ee13 100644
--- a/src/source-hive/src/main/java/org/apache/kylin/source/jdbc/H2Database.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/source/jdbc/H2Database.java
@@ -36,8 +36,8 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.NTableMetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/core-job/pom.xml b/src/core-metrics/pom.xml
similarity index 51%
copy from src/core-job/pom.xml
copy to src/core-metrics/pom.xml
index bbdbb0cf2f..a7e8742a4e 100644
--- a/src/core-job/pom.xml
+++ b/src/core-metrics/pom.xml
@@ -1,78 +1,83 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
+<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>kylin-core-job</artifactId>
- <packaging>jar</packaging>
- <name>Kylin - Core Job</name>
- <url>http://kylin.apache.org</url>
- <description>Kylin Core Job</description>
-
<parent>
- <groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
+ <groupId>org.apache.kylin</groupId>
<version>5.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
+ <name>Kylin - Core Metrics</name>
+ <url>http://kylin.apache.org</url>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-core-metrics</artifactId>
<properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
- <!-- provided -->
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
+ <artifactId>kylin-core-common</artifactId>
</dependency>
<dependency>
<groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-curator-test</artifactId>
+ <artifactId>kap-external-influxdb</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
</dependency>
-
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-json</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jmx</artifactId>
</dependency>
+
<dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.junit.vintage</groupId>
- <artifactId>junit-vintage-engine</artifactId>
- </exclusion>
- </exclusions>
</dependency>
- <!--Env & Test-->
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
@@ -84,30 +89,41 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.assertj</groupId>
- <artifactId>assertj-core</artifactId>
+ <groupId>org.junit-pioneer</groupId>
+ <artifactId>junit-pioneer</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.awaitility</groupId>
- <artifactId>awaitility</artifactId>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-inline</artifactId>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsController.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsController.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsController.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsController.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsName.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsName.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsName.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsName.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java
index 965a183567..5686c85f97 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java
+++ b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java
@@ -36,12 +36,12 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
-import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.metrics.MetricsCategory;
import org.apache.kylin.common.metrics.MetricsGroup;
import org.apache.kylin.common.metrics.MetricsName;
import org.apache.kylin.common.metrics.service.InfluxDBInstance;
import org.apache.kylin.common.util.AddressUtil;
+import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java
index fc821e01e3..3f8e96f31b 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java
+++ b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java
@@ -26,10 +26,10 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ExecutorServiceUtil;
-import org.apache.kylin.common.util.NamedThreadFactory;
import org.apache.kylin.common.metrics.MetricsGroup;
+import org.apache.kylin.common.util.ExecutorServiceUtil;
import org.apache.kylin.common.util.InfluxDBUtils;
+import org.apache.kylin.common.util.NamedThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java b/src/core-metrics/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java
similarity index 100%
rename from src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java
rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java
similarity index 100%
rename from src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java
rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java
similarity index 89%
rename from src/core-common/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java
rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java
index 64a1ba1a58..5786d4ed12 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java
+++ b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java
@@ -42,18 +42,18 @@ import org.springframework.security.core.context.SecurityContextHolder;
import io.kyligence.kap.shaded.influxdb.org.influxdb.InfluxDB;
@RunWith(PowerMockRunner.class)
-@PowerMockIgnore({"javax.net.ssl.*", "javax.management.*", "org.apache.hadoop.*", "javax.security.*", "javax.crypto.*", "javax.script.*"})
-@PrepareForTest({InfluxDBInstance.class, InfluxDBUtils.class, UserGroupInformation.class})
-public class MetricsControllerTest extends NLocalFileMetadataTestCase{
+@PowerMockIgnore({ "javax.net.ssl.*", "javax.management.*", "org.apache.hadoop.*", "javax.security.*", "javax.crypto.*",
+ "javax.script.*" })
+@PrepareForTest({ InfluxDBInstance.class, InfluxDBUtils.class, UserGroupInformation.class })
+public class MetricsControllerTest extends NLocalFileMetadataTestCase {
public final static String ROLE_ADMIN = "ROLE_ADMIN";
private final Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", ROLE_ADMIN);
-
@Before
public void setUp() throws Exception {
PowerMockito.mockStatic(UserGroupInformation.class);
PowerMockito.mockStatic(InfluxDBUtils.class);
-
+
UserGroupInformation userGroupInformation = Mockito.mock(UserGroupInformation.class);
PowerMockito.when(UserGroupInformation.getCurrentUser()).thenAnswer(invocation -> userGroupInformation);
overwriteSystemProp("HADOOP_USER_NAME", "root");
@@ -66,14 +66,14 @@ public class MetricsControllerTest extends NLocalFileMetadataTestCase{
cleanupTestMetadata();
}
-
@Test
public void initTest() throws Exception {
KapConfig config = KapConfig.wrap(KylinConfig.getInstanceFromEnv());
InfluxDB influxDB = Mockito.mock(InfluxDB.class);
- PowerMockito.doAnswer(invocationOnMock -> influxDB).when(InfluxDBUtils.class, "getInfluxDBInstance", Mockito.anyString(), Mockito.anyString(),
- Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyBoolean());
+ PowerMockito.doAnswer(invocationOnMock -> influxDB).when(InfluxDBUtils.class, "getInfluxDBInstance",
+ Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean());
Mockito.when(influxDB.databaseExists(Mockito.anyString())).thenReturn(true);
MetricsController.init(config);
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java
similarity index 100%
rename from src/core-common/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java
rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
index a9eadeff1d..b5a2cec688 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
@@ -68,7 +68,6 @@ import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -151,7 +150,7 @@ public class BaseController {
if (kylinException != null) {
cause = kylinException;
}
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause);
+ return new ErrorResponse(req.getRequestURL().toString(), cause);
}
@ResponseStatus(HttpStatus.FORBIDDEN)
@@ -159,7 +158,7 @@ public class BaseController {
@ResponseBody
ErrorResponse handleForbidden(HttpServletRequest req, Exception ex) {
getLogger().error("", ex);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
@ResponseStatus(HttpStatus.NOT_FOUND)
@@ -167,7 +166,7 @@ public class BaseController {
@ResponseBody
ErrorResponse handleNotFound(HttpServletRequest req, Exception ex) {
getLogger().error("", ex);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -191,7 +190,7 @@ public class BaseController {
ErrorResponse handleAccessDenied(HttpServletRequest req, Throwable ex) {
getLogger().error("", ex);
KylinException e = new KylinException(ACCESS_DENIED, MsgPicker.getMsg().getAccessDeny());
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e);
+ return new ErrorResponse(req.getRequestURL().toString(), e);
}
@ResponseStatus(HttpStatus.BAD_REQUEST)
@@ -200,7 +199,7 @@ public class BaseController {
ErrorResponse handleInvalidRequestParam(HttpServletRequest req, Throwable ex) {
KylinException e = new KylinException(INVALID_PARAMETER, ex);
getLogger().error("", e);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e);
+ return new ErrorResponse(req.getRequestURL().toString(), e);
}
@ResponseStatus(HttpStatus.BAD_REQUEST)
@@ -210,7 +209,7 @@ public class BaseController {
String parameterName = ex.getParameterName();
KylinException e = new KylinException(REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY, parameterName);
getLogger().error("", e);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e);
+ return new ErrorResponse(req.getRequestURL().toString(), e);
}
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -219,14 +218,14 @@ public class BaseController {
ErrorResponse handleErrorCode(HttpServletRequest req, Throwable ex) {
getLogger().error("", ex);
KylinException cause = (KylinException) ex;
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause);
+ return new ErrorResponse(req.getRequestURL().toString(), cause);
}
@ResponseStatus(HttpStatus.BAD_REQUEST)
@ExceptionHandler(MethodArgumentNotValidException.class)
@ResponseBody
ErrorResponse handleInvalidArgument(HttpServletRequest request, MethodArgumentNotValidException ex) {
- val response = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request), ex);
+ val response = new ErrorResponse(request.getRequestURL().toString(), ex);
val target = ex.getBindingResult().getTarget();
if (target instanceof Validation) {
response.setMsg(((Validation) target).getErrorMessage(ex.getBindingResult().getFieldErrors()));
@@ -244,7 +243,7 @@ public class BaseController {
ErrorResponse handleUnauthorized(HttpServletRequest req, Throwable ex) {
KylinException e = new KylinException(USER_UNAUTHORIZED, ex);
getLogger().error("", e);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
protected void checkRequiredArg(String fieldName, Object fieldValue) {
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java
index 42477bede6..63256b746b 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java
@@ -42,7 +42,6 @@ import org.apache.kylin.common.exception.ErrorCode;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.rest.response.ErrorResponse;
import org.springframework.core.annotation.Order;
import org.springframework.http.HttpHeaders;
@@ -84,7 +83,7 @@ public class SegmentsRequestFilter implements Filter {
MsgPicker.setMsg(httpServletRequest.getHeader(HttpHeaders.ACCEPT_LANGUAGE));
ErrorCode.setMsg(httpServletRequest.getHeader(HttpHeaders.ACCEPT_LANGUAGE));
- ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(httpServletRequest),
+ ErrorResponse errorResponse = new ErrorResponse(httpServletRequest.getRequestURL().toString(),
e);
byte[] responseBody = JsonUtil.writeValueAsBytes(errorResponse);
diff --git a/src/datasource-sdk/pom.xml b/src/datasource-sdk/pom.xml
index 27bcc81db4..992fac1baa 100644
--- a/src/datasource-sdk/pom.xml
+++ b/src/datasource-sdk/pom.xml
@@ -14,11 +14,6 @@
<name>Kylin - Datasource SDK</name>
<dependencies>
- <!-- provided -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-metadata</artifactId>
@@ -28,10 +23,6 @@
<artifactId>jackson-dataformat-xml</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-linq4j</artifactId>
- </dependency>
<!--Test and Environment-->
<dependency>
<groupId>org.apache.kylin</groupId>
@@ -40,14 +31,8 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-source-hive</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
<scope>test</scope>
</dependency>
<dependency>
@@ -56,13 +41,13 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/src/datasource-service/pom.xml b/src/datasource-service/pom.xml
index 69e2d8427b..acd5eacfa9 100644
--- a/src/datasource-service/pom.xml
+++ b/src/datasource-service/pom.xml
@@ -80,6 +80,10 @@
<artifactId>spark-sql_2.12</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>distributed-lock-ext</artifactId>
+ </dependency>
<!-- test -->
<dependency>
diff --git a/src/core-job/pom.xml b/src/distributed-lock-ext/pom.xml
similarity index 51%
copy from src/core-job/pom.xml
copy to src/distributed-lock-ext/pom.xml
index bbdbb0cf2f..3df312fd32 100644
--- a/src/core-job/pom.xml
+++ b/src/distributed-lock-ext/pom.xml
@@ -1,91 +1,79 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
+<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>kylin-core-job</artifactId>
- <packaging>jar</packaging>
- <name>Kylin - Core Job</name>
- <url>http://kylin.apache.org</url>
- <description>Kylin Core Job</description>
-
<parent>
- <groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
+ <groupId>org.apache.kylin</groupId>
<version>5.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>distributed-lock-ext</artifactId>
+ <name>Kylin - Distributed Lock Extension</name>
<properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
- <!-- provided -->
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
+ <artifactId>kylin-core-common</artifactId>
</dependency>
<dependency>
<groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-curator-test</artifactId>
+ <artifactId>kap-external-curator</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
+ <groupId>org.springframework.integration</groupId>
+ <artifactId>spring-integration-jdbc</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
+ <groupId>org.mybatis</groupId>
+ <artifactId>mybatis</artifactId>
</dependency>
-
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <groupId>org.mybatis.dynamic-sql</groupId>
+ <artifactId>mybatis-dynamic-sql</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-dbcp2</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.junit.vintage</groupId>
- <artifactId>junit-vintage-engine</artifactId>
- </exclusion>
- </exclusions>
</dependency>
- <!--Env & Test-->
<dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-common</artifactId>
+ <type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.junit.vintage</groupId>
- <artifactId>junit-vintage-engine</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.assertj</groupId>
- <artifactId>assertj-core</artifactId>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
@@ -94,20 +82,21 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-inline</artifactId>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
+ <groupId>org.junit-pioneer</groupId>
+ <artifactId>junit-pioneer</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
-</project>
+
+</project>
\ No newline at end of file
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java
rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java
rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java
rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java
similarity index 99%
rename from src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java
rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java
index be93e5da19..edd7eddb03 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java
+++ b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java
@@ -18,17 +18,6 @@
package org.apache.kylin.common.lock.jdbc;
-import lombok.extern.slf4j.Slf4j;
-import lombok.val;
-import org.apache.commons.dbcp2.BasicDataSource;
-import org.apache.ibatis.jdbc.ScriptRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.StorageURL;
-import org.apache.kylin.common.logging.LogOutputStream;
-import org.apache.kylin.common.persistence.metadata.JdbcDataSource;
-import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil;
-
-import javax.sql.DataSource;
import java.io.ByteArrayInputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
@@ -38,6 +27,19 @@ import java.sql.Connection;
import java.util.Locale;
import java.util.Properties;
+import javax.sql.DataSource;
+
+import org.apache.commons.dbcp2.BasicDataSource;
+import org.apache.ibatis.jdbc.ScriptRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.StorageURL;
+import org.apache.kylin.common.logging.LogOutputStream;
+import org.apache.kylin.common.persistence.metadata.JdbcDataSource;
+import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil;
+
+import lombok.val;
+import lombok.extern.slf4j.Slf4j;
+
@Slf4j
public class JdbcDistributedLockUtil {
private static final Charset DEFAULT_CHARSET = Charset.defaultCharset();
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/ZKUtil.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/util/ZKUtil.java
similarity index 96%
rename from src/core-common/src/main/java/org/apache/kylin/common/util/ZKUtil.java
rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/util/ZKUtil.java
index dbbeb9277a..6ab6e860ab 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/ZKUtil.java
+++ b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/util/ZKUtil.java
@@ -18,9 +18,6 @@
package org.apache.kylin.common.util;
-import static org.apache.kylin.common.ZookeeperConfig.geZKClientConnectionTimeoutMs;
-import static org.apache.kylin.common.ZookeeperConfig.geZKClientSessionTimeoutMs;
-
import java.io.File;
import java.io.IOException;
import java.util.concurrent.Callable;
@@ -28,6 +25,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.ZookeeperConfig;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -177,8 +175,8 @@ public class ZKUtil {
if (!Strings.isNullOrEmpty(zkRoot)) {
zkString += zkRoot;
}
- int sessionTimeout = geZKClientSessionTimeoutMs();
- int connectionTimeout = geZKClientConnectionTimeoutMs();
+ int sessionTimeout = ZookeeperConfig.geZKClientSessionTimeoutMs();
+ int connectionTimeout = ZookeeperConfig.geZKClientConnectionTimeoutMs();
return CuratorFrameworkFactory.newClient(zkString, sessionTimeout, connectionTimeout, retryPolicy);
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java
similarity index 98%
rename from src/core-common/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java
rename to src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java
index 2b774bcd41..ea90eed3ba 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java
+++ b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java
@@ -18,8 +18,8 @@
package org.apache.kylin.common.lock;
-import lombok.extern.slf4j.Slf4j;
-import org.junit.Assert;
+import static org.apache.kylin.common.util.TestUtils.getTestConfig;
+import static org.awaitility.Awaitility.await;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@@ -27,8 +27,9 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
-import static org.apache.kylin.common.util.TestUtils.getTestConfig;
-import static org.awaitility.Awaitility.await;
+import org.junit.Assert;
+
+import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DistributedLockFactoryTest {
@@ -39,7 +40,7 @@ public class DistributedLockFactoryTest {
final CountDownLatch tasks = new CountDownLatch(threadNum);
- int[] count = new int[] {0};
+ int[] count = new int[] { 0 };
for (int i = 0; i < threadNum; i++) {
executorService.submit(new DirtyReadTest(key, times, tasks, count));
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java
similarity index 73%
rename from src/core-common/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java
rename to src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java
index ea3692a8d0..cec38b22ca 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java
+++ b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java
@@ -18,9 +18,6 @@
package org.apache.kylin.common.lock.curator;
-import static org.apache.kylin.common.util.TestUtils.getTestConfig;
-import static org.awaitility.Awaitility.await;
-
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@@ -29,7 +26,9 @@ import java.util.concurrent.locks.Lock;
import org.apache.curator.test.TestingServer;
import org.apache.kylin.common.lock.DistributedLockFactoryTest;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.common.util.TestUtils;
import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.awaitility.Awaitility;
import org.junit.Assert;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -64,8 +63,9 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
public void testBasic() throws Exception {
String path = "/test/distributed_lock_factory_test/test_basic/" + RandomUtil.randomUUIDStr();
- getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
- CuratorDistributedLock lock = (CuratorDistributedLock) getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path);
+ TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
+ CuratorDistributedLock lock = (CuratorDistributedLock) TestUtils.getTestConfig().getDistributedLockFactory()
+ .getLockForCurrentThread(path);
Assert.assertFalse(lock.isAcquiredInThisThread());
lock.lock();
@@ -81,14 +81,15 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
ExecutorService executorService = Executors.newFixedThreadPool(1);
- getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
- getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1");
- getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000");
+ TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
+ TestUtils.getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1");
+ TestUtils.getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000");
executorService.submit(() -> {
CuratorDistributedLock lock = null;
try {
- lock = (CuratorDistributedLock)getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path);
+ lock = (CuratorDistributedLock) TestUtils.getTestConfig().getDistributedLockFactory()
+ .getLockForCurrentThread(path);
} catch (Exception e) {
e.printStackTrace();
}
@@ -103,7 +104,7 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
}
});
- await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
+ Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
Assert.assertFalse(isInterrupted);
locked = false;
@@ -111,15 +112,15 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
// zk for thread1 lost
// thread1 will be interrupted
- await().atMost(20, TimeUnit.SECONDS).until(() -> isInterrupted);
+ Awaitility.await().atMost(20, TimeUnit.SECONDS).until(() -> isInterrupted);
Assert.assertFalse(locked);
- getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer2.getConnectString());
+ TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer2.getConnectString());
executorService.submit(() -> {
Lock lock = null;
try {
- lock = getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path);
+ lock = TestUtils.getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path);
} catch (Exception e) {
e.printStackTrace();
}
@@ -130,7 +131,7 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
// thread1 released the lock
// thread2 will get the lock
- await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
+ Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
}
@Test
@@ -141,10 +142,10 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
ExecutorService executorService = Executors.newFixedThreadPool(1);
CuratorDistributedLockFactory lockFactory;
CuratorDistributedLock lock1;
- getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
- getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1");
- getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000");
- lockFactory = (CuratorDistributedLockFactory) getTestConfig().getDistributedLockFactory();
+ TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
+ TestUtils.getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1");
+ TestUtils.getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000");
+ lockFactory = (CuratorDistributedLockFactory) TestUtils.getTestConfig().getDistributedLockFactory();
lock1 = lockFactory.getLockForCurrentThread(path);
executorService.submit(() -> {
@@ -158,7 +159,7 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
}
});
- await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
+ Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
Assert.assertFalse(isInterrupted);
@@ -169,12 +170,12 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
// zk for thread1 suspended
// thread1 will be interrupted
- await().atMost(5, TimeUnit.SECONDS).until(() -> isInterrupted);
+ Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> isInterrupted);
}
@Test
void testConcurrence() throws Exception {
- getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
+ TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
String key = "/test/distributed_lock_factory_test/test_concurrence/" + RandomUtil.randomUUIDStr();
super.testConcurrence(key, 10, 10);
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java
similarity index 91%
rename from src/core-common/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java
rename to src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java
index 45e8b533a4..4e5af52c0b 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java
+++ b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java
@@ -18,26 +18,25 @@
package org.apache.kylin.common.lock.jdbc;
+import static org.apache.kylin.common.util.TestUtils.getTestConfig;
+
+import java.util.UUID;
+
import org.apache.kylin.common.lock.DistributedLockFactoryTest;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.junit.annotation.OverwriteProp;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import java.util.UUID;
-
-import static org.apache.kylin.common.util.TestUtils.getTestConfig;
-
@MetadataInfo(onlyProps = true)
class JdbcDistributedLockFactoryTest extends DistributedLockFactoryTest {
@BeforeEach
public void setup() {
- getTestConfig().getDistributedLockFactory().initialize();
+ getTestConfig().getDistributedLockFactory().initialize();
}
- @OverwriteProp(key = "kylin.metadata.distributed-lock-impl",
- value = "org.apache.kylin.common.lock.jdbc.JdbcDistributedLockFactory")
+ @OverwriteProp(key = "kylin.metadata.distributed-lock-impl", value = "org.apache.kylin.common.lock.jdbc.JdbcDistributedLockFactory")
@Test
void testConcurrence() throws Exception {
getTestConfig().getDistributedLockFactory().initialize();
diff --git a/src/kylin-it/pom.xml b/src/kylin-it/pom.xml
index 19a5def3a6..abd2acc15f 100644
--- a/src/kylin-it/pom.xml
+++ b/src/kylin-it/pom.xml
@@ -11,6 +11,11 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>kylin-it</artifactId>
+ <name>Kylin - Integration Test</name>
+
+ <properties>
+ <beanutils.version>1.9.4</beanutils.version>
+ </properties>
<dependencies>
<!--Springfox Swagger2 v2.9.2, swagger2markup v1.3.3. has specific requirement on guava and commons-beanutils -->
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java b/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java
index 59acd33a6f..59a6912cac 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java
@@ -384,6 +384,8 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
val entry = Mockito.spy(new StreamingMergeEntry());
val segments = new Segments<NDataSegment>();
+ val mgr = NDataflowManager.getInstance(getTestConfig(), PROJECT);
+ var dataflow = mgr.getDataflow(DATAFLOW_ID);
for (int i = 0; i < 3; i++) {
val start = LocalDate.parse("2000-01-01").plusMonths(i);
val end = start.plusMonths(1);
@@ -395,6 +397,7 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
seg.setSegmentRange(segRange);
seg.setStatus(SegmentStatusEnum.READY);
seg.setAdditionalInfo(null);
+ seg.setDataflow(dataflow);
segments.add(seg);
}
@@ -407,6 +410,8 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
val segments = new Segments<NDataSegment>();
val addInfo = new HashMap<String, String>();
+ val mgr = NDataflowManager.getInstance(getTestConfig(), PROJECT);
+ var dataflow = mgr.getDataflow(DATAFLOW_ID);
addInfo.put("abc", "2");
for (int i = 0; i < 3; i++) {
val start = LocalDate.parse("2000-01-01").plusMonths(i);
@@ -419,6 +424,7 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
seg.setSegmentRange(segRange);
seg.setStatus(SegmentStatusEnum.READY);
seg.setAdditionalInfo(addInfo);
+ seg.setDataflow(dataflow);
segments.add(seg);
}
@@ -431,6 +437,8 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
val segments = new Segments<NDataSegment>();
val addInfo = new HashMap<String, String>();
+ val mgr = NDataflowManager.getInstance(getTestConfig(), PROJECT);
+ var dataflow = mgr.getDataflow(DATAFLOW_ID);
addInfo.put(StreamingConstants.FILE_LAYER, "2");
for (int i = 0; i < 3; i++) {
val start = LocalDate.parse("2000-01-01").plusMonths(i);
@@ -443,6 +451,7 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
seg.setSegmentRange(segRange);
seg.setStatus(SegmentStatusEnum.READY);
seg.setAdditionalInfo(addInfo);
+ seg.setDataflow(dataflow);
segments.add(seg);
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java b/src/modeling-service/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java
rename to src/modeling-service/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java b/src/modeling-service/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java
rename to src/modeling-service/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
index 4decdbb834..f6866f2708 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
@@ -423,7 +423,7 @@ public class ModelSemanticHelper extends BasicService {
.forEach(x -> x.changeTableAlias(oldAliasName, newAliasName));
model.getAllMeasures().stream().filter(x -> !x.isTomb())
.forEach(x -> x.changeTableAlias(oldAliasName, newAliasName));
- model.getComputedColumnDescs().forEach(x -> x.changeTableAlias(oldAliasName, newAliasName));
+ model.getComputedColumnDescs().forEach(x -> changeTableAlias(x, oldAliasName, newAliasName));
String filterCondition = model.getFilterCondition();
if (StringUtils.isNotEmpty(filterCondition)) {
@@ -436,6 +436,13 @@ public class ModelSemanticHelper extends BasicService {
}
}
+ private void changeTableAlias(ComputedColumnDesc computedColumnDesc, String oldAlias, String newAlias) {
+ SqlVisitor<Object> modifyAlias = new ModifyTableNameSqlVisitor(oldAlias, newAlias);
+ SqlNode sqlNode = CalciteParser.getExpNode(computedColumnDesc.getExpression());
+ sqlNode.accept(modifyAlias);
+ computedColumnDesc.setExpression(sqlNode.toSqlString(HiveSqlDialect.DEFAULT).toString());
+ }
+
private Map<String, String> getAliasTransformMap(NDataModel originModel, NDataModel expectModel) {
Map<String, String> matchAlias = Maps.newHashMap();
boolean match = originModel.getJoinsGraph().match(expectModel.getJoinsGraph(), matchAlias);
diff --git a/src/query/pom.xml b/src/query/pom.xml
index 1479d73d1f..a970b1e41d 100644
--- a/src/query/pom.xml
+++ b/src/query/pom.xml
@@ -64,6 +64,10 @@
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-linq4j</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ </dependency>
<dependency>
<groupId>commons-collections</groupId>
diff --git a/src/server/pom.xml b/src/server/pom.xml
index 755e7c6ce4..d6a413ec2c 100644
--- a/src/server/pom.xml
+++ b/src/server/pom.xml
@@ -69,6 +69,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kap-second-storage-ui</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>distributed-lock-ext</artifactId>
+ </dependency>
<!-- https://mvnrepository.com/artifact/org.scala-lang.modules/scala-xml -->
<dependency>
@@ -177,6 +181,26 @@
<groupId>com.taobao.arthas</groupId>
<artifactId>arthas-spring-boot-starter</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.micrometer</groupId>
+ <artifactId>micrometer-registry-prometheus</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.latencyutils</groupId>
+ <artifactId>LatencyUtils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java b/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
index 54c07d3fbb..50e7276660 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
@@ -53,7 +53,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.Unsafe;
+import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.resourcegroup.ResourceGroupManager;
import org.apache.kylin.rest.cluster.ClusterManager;
@@ -76,7 +76,6 @@ import org.springframework.web.context.request.ServletRequestAttributes;
import com.google.common.collect.Sets;
-import org.apache.kylin.metadata.epoch.EpochManager;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -236,7 +235,7 @@ public class QueryNodeFilter implements Filter {
responseStatus = HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
Message msg = MsgPicker.getMsg();
KylinException exception = getKylinException(project, msg);
- ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(servletRequest),
+ ErrorResponse errorResponse = new ErrorResponse(servletRequest.getRequestURL().toString(),
exception);
responseBody = JsonUtil.writeValueAsBytes(errorResponse);
responseHeaders = new HttpHeaders();
@@ -358,7 +357,7 @@ public class QueryNodeFilter implements Filter {
public void writeConnectionErrorResponse(HttpServletRequest servletRequest, HttpServletResponse servletResponse)
throws IOException {
- ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(servletRequest),
+ ErrorResponse errorResponse = new ErrorResponse(servletRequest.getRequestURL().toString(),
new KylinException(FAILED_CONNECT_CATALOG, MsgPicker.getMsg().getConnectDatabaseError(), false));
byte[] responseBody = JsonUtil.writeValueAsBytes(errorResponse);
HttpHeaders responseHeaders = new HttpHeaders();
diff --git a/src/spark-project/engine-build-sdk/pom.xml b/src/spark-project/engine-build-sdk/pom.xml
index 0e2b3d6866..da144f3e7d 100644
--- a/src/spark-project/engine-build-sdk/pom.xml
+++ b/src/spark-project/engine-build-sdk/pom.xml
@@ -29,7 +29,6 @@
<name>Kylin - Engine Build SDK</name>
<artifactId>kylin-engine-build-sdk</artifactId>
-
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
diff --git a/src/spark-project/engine-spark/pom.xml b/src/spark-project/engine-spark/pom.xml
index b03cffa85b..2b4edf32e3 100644
--- a/src/spark-project/engine-spark/pom.xml
+++ b/src/spark-project/engine-spark/pom.xml
@@ -62,6 +62,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-engine-build-sdk</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>distributed-lock-ext</artifactId>
+ </dependency>
<dependency>
<groupId>io.kyligence.ke</groupId>
diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
index 145e017504..805e8ae20f 100644
--- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
+++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
@@ -35,6 +35,9 @@ import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.engine.spark.ExecutableUtils;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableParams;
@@ -42,11 +45,6 @@ import org.apache.kylin.job.execution.NExecutableManager;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
import org.apache.kylin.job.manager.JobManager;
import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.model.SegmentRange;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.common.persistence.transaction.UnitOfWork;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
-import org.apache.kylin.engine.spark.ExecutableUtils;
import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
@@ -60,6 +58,8 @@ import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.kylin.metadata.cube.model.PartitionStatusEnum;
import org.apache.kylin.metadata.job.JobBucket;
import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.SegmentRange;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -353,17 +353,18 @@ public class JobManagerTest extends NLocalFileMetadataTestCase {
@Test
public void testQuotaLimitReached() {
- thrown.expect(KylinException.class);
NDefaultScheduler defaultScheduler = NDefaultScheduler.getInstance(PROJECT);
- defaultScheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
+ defaultScheduler.init(new JobEngineConfig(getTestConfig()));
defaultScheduler.getContext().setReachQuotaLimit(true);
- JobParam param = new JobParam(Sets.newHashSet(), null, null, "ADMIn", Sets.newHashSet(), null);
- try {
- jobManager.addJob(param);
- } finally {
- defaultScheduler.forceShutdown();
- defaultScheduler.getContext().setReachQuotaLimit(false);
- }
+ Assert.assertThrows(KylinException.class, () -> {
+ try {
+ JobManager.checkStorageQuota(PROJECT);
+ } finally {
+ defaultScheduler.forceShutdown();
+ defaultScheduler.getContext().setReachQuotaLimit(false);
+ }
+ });
+
}
@Test
diff --git a/src/spark-project/spark-common/pom.xml b/src/spark-project/spark-common/pom.xml
index ff80077128..9556bd21d1 100644
--- a/src/spark-project/spark-common/pom.xml
+++ b/src/spark-project/spark-common/pom.xml
@@ -140,6 +140,11 @@
<artifactId>curator-test</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-soft-affinity-cache</artifactId>
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
similarity index 94%
rename from src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
index a0ba5f13a1..1c0b3a795e 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
+++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
@@ -47,18 +47,6 @@ public class AsyncProfiler {
return profiler;
}
- // for ut test
- public static AsyncProfiler utInstance() {
- if (profiler == null) {
- return new AsyncProfiler("");
- }
- return profiler;
- }
-
- private AsyncProfiler(String ignore) {
- logger.info("Test arg for ut: {}", ignore);
- }
-
private AsyncProfiler(boolean loadLocalLib) {
try {
boolean isTestingOnLocalMac = System.getProperty("os.name", "").contains("Mac")
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java
rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java
similarity index 100%
rename from src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java
rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java
similarity index 98%
rename from src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java
rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java
index 0a2d2a2ffb..17f63712ae 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java
+++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java
@@ -204,7 +204,7 @@ public class DFSFileTableReader implements TableReader {
String nextLine() throws IOException; // return null on EOF
}
- private class SeqRowReader implements RowReader {
+ private static class SeqRowReader implements RowReader {
Reader reader;
Writable key;
Text value;
@@ -230,7 +230,7 @@ public class DFSFileTableReader implements TableReader {
}
}
- private class CsvRowReader implements RowReader {
+ private static class CsvRowReader implements RowReader {
BufferedReader reader;
CsvRowReader(FileSystem fs, String path) throws IOException {
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so
new file mode 100644
index 0000000000..b959823506
Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so differ
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so
new file mode 100644
index 0000000000..6d961cec0d
Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so differ
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so
new file mode 100644
index 0000000000..ad45237d5f
Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so differ
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so
new file mode 100755
index 0000000000..4153f52868
Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so differ
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so
new file mode 100755
index 0000000000..c7298c6b82
Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so differ
diff --git a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala
similarity index 100%
rename from src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala
rename to src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala
diff --git a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
similarity index 99%
rename from src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
rename to src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
index 69bb7c72bb..d618896946 100644
--- a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
@@ -20,7 +20,7 @@ package org.apache.kylin.common.asyncprofiler
import org.slf4j.{Logger, LoggerFactory}
-/**
+/*
* this class is not thread safe
*/
object AsyncProfilerTool {
diff --git a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala
similarity index 98%
rename from src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala
rename to src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala
index cd3019d528..881992904b 100644
--- a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala
@@ -18,7 +18,8 @@
package org.apache.kylin.common.asyncprofiler
-/** simple command in string
+/*
+ * simple command in string
* messages are in form of {command}{executor-id}:{param}
* commands are limited to fixed 3 characters
*/
@@ -68,4 +69,3 @@ object Message {
(getCommand(msg), getId(msg), getParam(msg))
}
}
-
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
similarity index 100%
rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
similarity index 79%
rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
index 176d9ad119..c819afc4f5 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
+++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
@@ -67,22 +67,4 @@ public class AsyncProfilerTest {
AsyncProfiler.getInstance(true)::stop);
}
- @Test
- public void testAsyncProfilerUtInstance() {
- AsyncProfiler originInstance = AsyncProfiler.getInstance(true);
- AsyncProfiler utInstance = AsyncProfiler.utInstance();
- Assert.assertSame(originInstance, utInstance);
- }
-
- @Test
- public void testStopUnload() {
- AsyncProfiler asyncProfiler = AsyncProfiler.utInstance();
- Assert.assertFalse(asyncProfiler.isLoaded());
- asyncProfiler.stop();
- }
-
- @Test
- public void testExecuteUnload() throws IOException {
- Assert.assertEquals("", AsyncProfiler.utInstance().execute(""));
- }
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
similarity index 100%
rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
similarity index 96%
rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
index fba1ad54a6..90c4423b69 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
+++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
@@ -18,7 +18,6 @@
package org.apache.kylin.common.asyncprofiler;
-import static org.awaitility.Awaitility.await;
import static org.mockito.Mockito.mock;
import java.io.File;
@@ -30,6 +29,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import org.awaitility.Awaitility;
import org.awaitility.Duration;
import org.junit.Assert;
import org.junit.Test;
@@ -82,7 +82,7 @@ public class AsyncProfilerUtilsTest {
ExecutorService cachedThreadPool = Executors.newCachedThreadPool();
Thread t1 = new Thread(() -> {
- await().pollDelay(new Duration(1, TimeUnit.SECONDS)).until(() -> true);
+ Awaitility.await().pollDelay(new Duration(1, TimeUnit.SECONDS)).until(() -> true);
for (int i = 0; i < 3; i++) {
cachedThreadPool.execute(() -> asyncProfilerUtils.cachedResult.countDown());
}
@@ -100,7 +100,7 @@ public class AsyncProfilerUtilsTest {
ExecutorService cachedThreadPool = Executors.newCachedThreadPool();
Thread t1 = new Thread(() -> {
- await().pollDelay(new Duration(1, TimeUnit.MILLISECONDS)).until(() -> true);
+ Awaitility.await().pollDelay(new Duration(1, TimeUnit.MILLISECONDS)).until(() -> true);
for (int i = 0; i < 3; i++) {
cachedThreadPool.execute(() -> asyncProfilerUtils.cachedResult.countDown());
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java
similarity index 96%
rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java
rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java
index eef0290854..dddaeaae68 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java
+++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java
@@ -37,7 +37,8 @@ public class MessageTest {
@Test
public void testExecutorMessage() {
- String executorMessage = Message.createExecutorMessage(Message.START(), "-0", AsyncProfilerToolTest.DUMP_PARAMS);
+ String executorMessage = Message.createExecutorMessage(Message.START(), "-0",
+ AsyncProfilerToolTest.DUMP_PARAMS);
Tuple3<String, String, String> tuple = Message.processMessage(executorMessage);
Assert.assertEquals("STA-0:flamegraph", executorMessage);
Assert.assertEquals(Message.START(), tuple._1());
diff --git a/src/systools/pom.xml b/src/systools/pom.xml
index c9026a1dac..f6e3cd1530 100644
--- a/src/systools/pom.xml
+++ b/src/systools/pom.xml
@@ -189,26 +189,11 @@
<artifactId>mapstruct</artifactId>
</dependency>
- <!-- Swagger -->
- <dependency>
- <groupId>io.springfox</groupId>
- <artifactId>springfox-boot-starter</artifactId>
- </dependency>
- <dependency>
- <groupId>io.springfox</groupId>
- <artifactId>springfox-boot-starter</artifactId>
- <version>3.0.0</version>
- </dependency>
-
- <dependency>
- <groupId>org.mapstruct</groupId>
- <artifactId>mapstruct</artifactId>
- </dependency>
-
<!-- util -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
@@ -230,6 +215,11 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>