You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/06/18 13:56:30 UTC
[kylin] 04/04: Rebase master
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master-hadoop3.1
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 21967133c6bb870cdeaa4ec660bfc5f951da231c
Author: nichunen <ni...@apache.org>
AuthorDate: Tue Jun 18 21:56:07 2019 +0800
Rebase master
---
pom.xml | 55 +++++++++++++++-------
server-base/pom.xml | 5 --
.../apache/kylin/rest/job/KylinHealthCheckJob.java | 11 +++--
.../org/apache/kylin/rest/service/CubeService.java | 6 +--
.../kylin/storage/hbase/util/StorageCleanUtil.java | 13 ++---
5 files changed, 54 insertions(+), 36 deletions(-)
diff --git a/pom.xml b/pom.xml
index ab6cf6a..83392d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -16,14 +16,15 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>19</version>
- <relativePath />
+ <relativePath/>
<!-- no parent resolution -->
</parent>
@@ -122,6 +123,7 @@
<jetty.version>9.3.22.v20171030</jetty.version>
<jamm.version>0.3.1</jamm.version>
<mockito.version>2.7.14</mockito.version>
+ <mockito-all.version>1.9.5</mockito-all.version>
<powermock.version>1.7.0</powermock.version>
<!-- Commons -->
@@ -129,6 +131,7 @@
<commons-email.version>1.5</commons-email.version>
<commons-validator.version>1.4.0</commons-validator.version>
<commons-compress>1.18</commons-compress>
+ <commons-dbcp.version>1.4</commons-dbcp.version>
<!-- Utility -->
<log4j.version>1.2.17</log4j.version>
@@ -146,6 +149,8 @@
<t-digest.version>3.1</t-digest.version>
<freemarker.version>2.3.23</freemarker.version>
<rocksdb.version>5.9.2</rocksdb.version>
+ <lz4.version>1.3.0</lz4.version>
+ <mssql-jdbc.version>6.2.2.jre8</mssql-jdbc.version>
<!--metric-->
<dropwizard.version>3.1.2</dropwizard.version>
<!-- REST Service, ref https://github.com/spring-projects/spring-boot/blob/v1.3.8.RELEASE/spring-boot-dependencies/pom.xml -->
@@ -398,6 +403,33 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-stream-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-stream-coordinator</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-stream-source-kafka</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-storage-stream</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-stream-receiver</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-datasource-sdk</artifactId>
@@ -901,12 +933,6 @@
<artifactId>opensaml</artifactId>
<version>${opensaml.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-test</artifactId>
- <version>2.12.0</version>
- <scope>test</scope>
- </dependency>
<!-- Spring Core -->
<dependency>
@@ -970,17 +996,15 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>${jetty.version}</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-webapp</artifactId>
+ <artifactId>jetty-servlet</artifactId>
<version>${jetty.version}</version>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
+ <artifactId>jetty-webapp</artifactId>
<version>${jetty.version}</version>
<scope>test</scope>
</dependency>
@@ -1020,11 +1044,6 @@
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
</dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-reflect</artifactId>
- <version>${scala.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.curator</groupId>
@@ -1266,7 +1285,7 @@
</goals>
</pluginExecutionFilter>
<action>
- <ignore />
+ <ignore/>
</action>
</pluginExecution>
</pluginExecutions>
diff --git a/server-base/pom.xml b/server-base/pom.xml
index 123effe..9d68add 100644
--- a/server-base/pom.xml
+++ b/server-base/pom.xml
@@ -248,11 +248,6 @@
<artifactId>jetty-webapp</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>junit</groupId>
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/KylinHealthCheckJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/KylinHealthCheckJob.java
index ecca373..d58cd9b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/KylinHealthCheckJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/KylinHealthCheckJob.java
@@ -24,8 +24,9 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.BufferedLogger;
@@ -44,6 +45,7 @@ import org.apache.kylin.job.execution.CheckpointExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -198,12 +200,13 @@ public class KylinHealthCheckJob extends AbstractApplication {
private void checkHBaseTables(List<CubeInstance> cubes) throws IOException {
reporter.log("## Checking HBase Table of segments");
- HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create());
+ Connection conn = HBaseConnection.get(config.getStorageUrl());
+ Admin hbaseAdmin = conn.getAdmin();
for (CubeInstance cube : cubes) {
for (CubeSegment segment : cube.getSegments()) {
if (segment.getStatus() != SegmentStatusEnum.NEW) {
String tableName = segment.getStorageLocationIdentifier();
- if ((!hbaseAdmin.tableExists(tableName)) || (!hbaseAdmin.isTableEnabled(tableName))) {
+ if ((!hbaseAdmin.tableExists(TableName.valueOf(tableName))) || (!hbaseAdmin.isTableEnabled(TableName.valueOf(tableName)))) {
reporter.log("HBase table: {} not exist for segment: {}, project: {}", tableName, segment,
cube.getProject());
reporter.log(
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 7378165..a981ca9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -29,7 +29,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -623,8 +623,8 @@ public class CubeService extends BasicService implements InitializingBean {
toDropHTables.add(seg.getStorageLocationIdentifier());
toDelHDFSPaths.add(JobBuilderSupport.getJobWorkingDir(seg.getConfig().getHdfsWorkingDirectory(), seg.getLastBuildJobID()));
}
-
- StorageCleanUtil.dropHTables(new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration()), toDropHTables);
+ Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
+ StorageCleanUtil.dropHTables(conn.getAdmin(), toDropHTables);
StorageCleanUtil.deleteHDFSPath(HadoopUtil.getWorkingFileSystem(), toDelHDFSPaths);
}
}
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanUtil.java
index a1259b8..0cae1e7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanUtil.java
@@ -21,7 +21,8 @@ package org.apache.kylin.storage.hbase.util;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,18 +36,18 @@ public class StorageCleanUtil {
/**
* this method will close hbaseAdmin after finishing the work.
*/
- public static void dropHTables(final HBaseAdmin hbaseAdmin, List<String> hTables) {
+ public static void dropHTables(final Admin hbaseAdmin, List<String> hTables) {
runSingleThreadTaskQuietly(() -> {
try {
for (String htable : hTables) {
logger.info("Deleting HBase table {}", htable);
- if (hbaseAdmin.tableExists(htable)) {
- if (hbaseAdmin.isTableEnabled(htable)) {
- hbaseAdmin.disableTable(htable);
+ if (hbaseAdmin.tableExists(TableName.valueOf(htable))) {
+ if (hbaseAdmin.isTableEnabled(TableName.valueOf(htable))) {
+ hbaseAdmin.disableTable(TableName.valueOf(htable));
}
- hbaseAdmin.deleteTable(htable);
+ hbaseAdmin.deleteTable(TableName.valueOf(htable));
logger.info("Deleted HBase table {}", htable);
} else {
logger.info("HBase table {} does not exist.", htable);