You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/03/03 04:31:03 UTC
[16/25] kylin git commit: KYLIN-1311 fix CI
KYLIN-1311 fix CI
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1ccbb38
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1ccbb38
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1ccbb38
Branch: refs/heads/helix-rebase
Commit: a1ccbb3825b5c24f6e43bced4ee5e27dff19d98b
Parents: f778ab4
Author: shaofengshi <sh...@apache.org>
Authored: Wed Feb 10 21:23:35 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Mar 2 17:27:46 2016 +0800
----------------------------------------------------------------------
.../rest/controller/ClusterController.java | 6 +++-
.../rest/controller/StreamingController.java | 6 ++--
.../helix/StreamCubeBuildTransitionHandler.java | 12 +++++--
.../apache/kylin/rest/service/CubeService.java | 11 ++++--
.../kylin/rest/service/StreamingService.java | 13 ++++---
.../rest/controller/UserControllerTest.java | 12 ++-----
.../kylin/rest/helix/HelixClusterAdminTest.java | 6 +++-
.../kylin/rest/service/CacheServiceTest.java | 2 --
.../kylin/rest/service/ServiceTestBase.java | 36 ++++++++------------
.../rest/service/TestBaseWithZookeeper.java | 9 ++---
10 files changed, 56 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/main/java/org/apache/kylin/rest/controller/ClusterController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/ClusterController.java b/server/src/main/java/org/apache/kylin/rest/controller/ClusterController.java
index 86a0398..51db0a7 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/ClusterController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/ClusterController.java
@@ -25,6 +25,7 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.streaming.StreamingManager;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
+import org.apache.kylin.job.lock.JobLock;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.helix.HelixClusterAdmin;
import org.apache.kylin.rest.request.StreamingBuildRequest;
@@ -32,6 +33,7 @@ import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@@ -48,6 +50,8 @@ import java.util.Collection;
public class ClusterController extends BasicController implements InitializingBean {
private static final Logger logger = LoggerFactory.getLogger(ClusterController.class);
+ @Autowired
+ private JobLock jobLock;
/*
* (non-Javadoc)
*
@@ -78,7 +82,7 @@ public class ClusterController extends BasicController implements InitializingBe
public void run() {
try {
DefaultScheduler scheduler = DefaultScheduler.createInstance();
- scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
+ scheduler.init(new JobEngineConfig(kylinConfig), jobLock);
if (!scheduler.hasStarted()) {
logger.error("scheduler has not been started");
System.exit(1);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index e33a1c9..74b0dae 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -253,8 +253,10 @@ public class StreamingController extends BasicController {
streamingBuildRequest.setStreaming(streamingConfig.getName());
try {
streamingService.buildStream(cube, streamingBuildRequest);
- } catch (IOException e) {
- e.printStackTrace();
+ } catch (IOException e) {
+ logger.error("", e);
+ streamingBuildRequest.setSuccessful(false);
+ streamingBuildRequest.setMessage("Failed to submit job for " + streamingBuildRequest.getStreaming() + ", error is: " + e.getMessage());
return streamingBuildRequest;
}
streamingBuildRequest.setMessage("Build request is submitted successfully.");
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/main/java/org/apache/kylin/rest/helix/StreamCubeBuildTransitionHandler.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/helix/StreamCubeBuildTransitionHandler.java b/server/src/main/java/org/apache/kylin/rest/helix/StreamCubeBuildTransitionHandler.java
index 705d8a7..4652d0d 100644
--- a/server/src/main/java/org/apache/kylin/rest/helix/StreamCubeBuildTransitionHandler.java
+++ b/server/src/main/java/org/apache/kylin/rest/helix/StreamCubeBuildTransitionHandler.java
@@ -2,6 +2,7 @@ package org.apache.kylin.rest.helix;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
+import org.apache.commons.io.IOUtils;
import org.apache.helix.NotificationContext;
import org.apache.helix.api.TransitionHandler;
import org.apache.helix.model.Message;
@@ -120,17 +121,22 @@ public class StreamCubeBuildTransitionHandler extends TransitionHandler {
private void runCMD(String cmd) {
logger.info("Executing: " + cmd);
+ BufferedReader input = null;
+ Process p = null;
try {
String line;
- Process p = Runtime.getRuntime().exec(cmd);
- BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()));
+ p = Runtime.getRuntime().exec(cmd);
+ input = new BufferedReader(new InputStreamReader(p.getInputStream()));
while ((line = input.readLine()) != null) {
logger.info(line);
}
- input.close();
+
+ logger.info("Successfully start: " + cmd);
} catch (IOException err) {
logger.error("Error happens when running '" + cmd + "'", err);
throw new RuntimeException(err);
+ } finally {
+ IOUtils.closeQuietly(input);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index e7411a9..b2a278a 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -590,10 +590,15 @@ public class CubeService extends BasicService {
public void updateOnNewSegmentReady(String cubeName) {
logger.debug("on updateOnNewSegmentReady: " + cubeName);
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- boolean isLeaderRole = true;
+ boolean isLeaderRole = false;
if (kylinConfig.isClusterEnabled()) {
- HelixClusterAdmin jobEngineAdmin = HelixClusterAdmin.getInstance(kylinConfig);
- isLeaderRole = jobEngineAdmin.isLeaderRole(HelixClusterAdmin.RESOURCE_NAME_JOB_ENGINE);
+ HelixClusterAdmin clusterAdmin = HelixClusterAdmin.getInstance(kylinConfig);
+ isLeaderRole = clusterAdmin.isLeaderRole(HelixClusterAdmin.RESOURCE_NAME_JOB_ENGINE);
+ } else {
+ String serverMode = kylinConfig.getServerMode();
+ if (Constant.SERVER_MODE_JOB.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase())) {
+ isLeaderRole = true;
+ }
}
logger.debug("server is leader role ? " + isLeaderRole);
if (isLeaderRole == true) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java b/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
index 6e732d9..28b9472 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/StreamingService.java
@@ -101,14 +101,13 @@ public class StreamingService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
public void buildStream(CubeInstance cube, StreamingBuildRequest streamingBuildRequest) throws IOException {
- HelixClusterAdmin clusterAdmin = HelixClusterAdmin.getInstance(KylinConfig.getInstanceFromEnv());
- try {
- clusterAdmin.addStreamingJob(streamingBuildRequest);
- } catch (IOException e) {
- logger.error("", e);
- streamingBuildRequest.setSuccessful(false);
- streamingBuildRequest.setMessage("Failed to submit job for " + streamingBuildRequest.getStreaming());
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ if (kylinConfig.isClusterEnabled() == false) {
+ throw new IllegalStateException("Set kylin.cluster.enabled to true to enable streaming feature.");
}
+
+ HelixClusterAdmin clusterAdmin = HelixClusterAdmin.getInstance(kylinConfig);
+ clusterAdmin.addStreamingJob(streamingBuildRequest);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
index fe0e67a..2c4a1cb 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
@@ -18,22 +18,14 @@
package org.apache.kylin.rest.controller;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.kylin.rest.service.ServiceTestBase;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
-import org.springframework.security.authentication.TestingAuthenticationToken;
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.GrantedAuthority;
-import org.springframework.security.core.context.SecurityContextHolder;
-import org.springframework.security.core.userdetails.User;
import org.springframework.security.core.userdetails.UserDetails;
+import java.io.IOException;
+
/**
* @author xduo
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/test/java/org/apache/kylin/rest/helix/HelixClusterAdminTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/helix/HelixClusterAdminTest.java b/server/src/test/java/org/apache/kylin/rest/helix/HelixClusterAdminTest.java
index 1c8b779..24a8bb3 100644
--- a/server/src/test/java/org/apache/kylin/rest/helix/HelixClusterAdminTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/helix/HelixClusterAdminTest.java
@@ -22,6 +22,7 @@ import org.I0Itec.zkclient.ZkClient;
import org.I0Itec.zkclient.ZkServer;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.helix.HelixAdmin;
import org.apache.helix.manager.zk.ZKHelixAdmin;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
@@ -44,6 +45,7 @@ public class HelixClusterAdminTest extends TestBaseWithZookeeper {
HelixClusterAdmin clusterAdmin1;
HelixClusterAdmin clusterAdmin2;
+ HelixAdmin zkHelixAdmin;
KylinConfig kylinConfig;
private static final String CLUSTER_NAME = "test_cluster";
@@ -54,7 +56,7 @@ public class HelixClusterAdminTest extends TestBaseWithZookeeper {
kylinConfig.setRestAddress("localhost:7070");
kylinConfig.setClusterName(CLUSTER_NAME);
- final ZKHelixAdmin zkHelixAdmin = new ZKHelixAdmin(zkAddress);
+ zkHelixAdmin = new ZKHelixAdmin(zkAddress);
zkHelixAdmin.dropCluster(kylinConfig.getClusterName());
}
@@ -117,6 +119,8 @@ public class HelixClusterAdminTest extends TestBaseWithZookeeper {
if (clusterAdmin2 != null) {
clusterAdmin2.stop();
}
+
+ zkHelixAdmin.dropCluster(CLUSTER_NAME);
cleanupTestMetadata();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index 8193884..c347219 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -52,8 +52,6 @@
//
// private static Server server;
//
-// private static String ZK_ADDRESS = "localhost:2199";
-//
// private static KylinConfig configA;
// private static KylinConfig configB;
//
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
index ca4fe39..020b2d0 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
@@ -18,12 +18,6 @@
package org.apache.kylin.rest.service;
-import com.google.common.collect.Lists;
-import org.I0Itec.zkclient.IDefaultNameSpace;
-import org.I0Itec.zkclient.ZkClient;
-import org.I0Itec.zkclient.ZkServer;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
@@ -32,42 +26,40 @@ import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationRegistry;
-import org.apache.kylin.rest.helix.HelixClusterAdmin;
import org.junit.*;
import org.junit.runner.RunWith;
import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.Authentication;
+import org.springframework.security.core.authority.AuthorityUtils;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.security.core.userdetails.User;
-import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
-import java.io.File;
-import java.util.Arrays;
-import java.util.List;
-
/**
* @author xduo
*/
@RunWith(SpringJUnit4ClassRunner.class)
-@ContextConfiguration(locations = { "classpath:applicationContext.xml", "classpath:kylinSecurity.xml" })
+@ContextConfiguration(locations = {"classpath:applicationContext.xml", "classpath:kylinSecurity.xml"})
@ActiveProfiles("testing")
-public class ServiceTestBase extends TestBaseWithZookeeper {
+public class ServiceTestBase extends LocalFileMetadataTestCase {
+
+ @BeforeClass
+ public static void setupResource() throws Exception {
+ staticCreateTestMetadata();
+ Authentication authentication = new TestingAuthenticationToken(new User("ADMIN", "ADMIN", AuthorityUtils.createAuthorityList("ROLE_ADMIN")), "ADMIN", "ROLE_ADMIN");
+ SecurityContextHolder.getContext().setAuthentication(authentication);
+ }
+
+ @AfterClass
+ public static void tearDownResource() {
+ }
@Before
public void setup() throws Exception {
this.createTestMetadata();
- UserService.UserGrantedAuthority userGrantedAuthority = new UserService.UserGrantedAuthority();
- userGrantedAuthority.setAuthority("ROLE_ADMIN");
- UserDetails user = new User("ADMIN", "skippped-ldap", Lists.newArrayList(userGrantedAuthority));
- Authentication authentication = new TestingAuthenticationToken(user, "ADMIN", "ROLE_ADMIN");
- SecurityContextHolder.getContext().setAuthentication(authentication);
- KylinConfig kylinConfig = this.getTestConfig();
- kylinConfig.setRestAddress("localhost:7070");
-
MetadataManager.clearCache();
CubeDescManager.clearCache();
CubeManager.clearCache();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a1ccbb38/server/src/test/java/org/apache/kylin/rest/service/TestBaseWithZookeeper.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/TestBaseWithZookeeper.java b/server/src/test/java/org/apache/kylin/rest/service/TestBaseWithZookeeper.java
index 3182c16..e51e632 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/TestBaseWithZookeeper.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/TestBaseWithZookeeper.java
@@ -25,9 +25,6 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.springframework.security.authentication.TestingAuthenticationToken;
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.context.SecurityContextHolder;
import java.io.File;
@@ -35,7 +32,7 @@ import java.io.File;
*/
public class TestBaseWithZookeeper extends LocalFileMetadataTestCase {
protected static final String zkAddress = "localhost:2199";
- static ZkServer server;
+ static ZkServer server = null;
static boolean zkStarted = false;
@BeforeClass
@@ -57,13 +54,13 @@ public class TestBaseWithZookeeper extends LocalFileMetadataTestCase {
zkStarted = true;
System.setProperty("kylin.zookeeper.address", zkAddress);
}
-
}
@AfterClass
public static void tearDownResource() {
- if (server == null) {
+ if (server != null) {
server.shutdown();
+ server = null;
zkStarted = false;
System.setProperty("kylin.zookeeper.address", "");
}