You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/02/14 03:53:20 UTC
[04/15] kylin git commit: rebase 2.x-staging
rebase 2.x-staging
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8882ad26
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8882ad26
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8882ad26
Branch: refs/heads/helix-201602
Commit: 8882ad26460f0e08c9f76fa8760c6d8e268419b5
Parents: ad05b21
Author: shaofengshi <sh...@apache.org>
Authored: Wed Dec 30 14:22:35 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Sat Feb 6 13:31:49 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 4 +++
.../kylin/rest/controller/JobController.java | 26 +++++++-------------
.../apache/kylin/rest/service/CubeService.java | 8 +++---
3 files changed, 18 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8882ad26/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 81f5827..ea77e47 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -264,5 +264,9 @@ public class KylinConfig extends KylinConfigBase {
out.println(key + "=" + val);
}
}
+
+ public String getClusterName() {
+ return this.getOptional("kylin.cluster.name", getMetadataUrlPrefix());
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8882ad26/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/JobController.java b/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
index 4d0824a..9dfb594 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -26,18 +26,15 @@ import java.util.List;
import java.util.Map;
import java.util.TimeZone;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import joptsimple.internal.Strings;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.restclient.Broadcaster;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.constant.JobTimeFilterEnum;
-import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
-import org.apache.kylin.job.lock.JobLock;
-import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.helix.HelixJobEngineAdmin;
import org.apache.kylin.rest.request.JobListRequest;
import org.apache.kylin.rest.service.JobService;
import org.slf4j.Logger;
@@ -50,6 +47,9 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
+import java.io.IOException;
+import java.util.*;
+
/**
* @author ysong1
* @author Jack
@@ -63,9 +63,6 @@ public class JobController extends BasicController implements InitializingBean {
@Autowired
private JobService jobService;
- @Autowired
- private JobLock jobLock;
-
/*
* (non-Javadoc)
*
@@ -79,15 +76,9 @@ public class JobController extends BasicController implements InitializingBean {
TimeZone tzone = TimeZone.getTimeZone(timeZone);
TimeZone.setDefault(tzone);
- if (System.getProperty("kylin.rest.address") == null) {
- throw new RuntimeException("There is no -Dkylin.rest.address set; Please check bin/kylin.sh");
- }
-
- final String restAddress = System.getProperty("kylin.rest.address");
- final String hostname = Preconditions.checkNotNull(restAddress.substring(0, restAddress.lastIndexOf(":")));
- final String port = Preconditions.checkNotNull(restAddress.substring(restAddress.lastIndexOf(":") + 1));
- final String instanceName = hostname + "_" + port;
+ final String instanceName = HelixJobEngineAdmin.getCurrentInstanceName();
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+
}
/**
@@ -199,7 +190,7 @@ public class JobController extends BasicController implements InitializingBean {
this.jobService = jobService;
}
- private void updateKylinConfig(List<String> instances) {
+ private void updateKylinCluster(List<String> instances) {
List<String> instanceRestAddresses = Lists.newArrayList();
for (String instanceName : instances) {
int indexOfUnderscore = instanceName.lastIndexOf("_");
@@ -208,6 +199,7 @@ public class JobController extends BasicController implements InitializingBean {
String restServersInCluster = Strings.join(instanceRestAddresses, ",");
KylinConfig.getInstanceFromEnv().setProperty("kylin.rest.servers", restServersInCluster);
System.setProperty("kylin.rest.servers", restServersInCluster);
+ Broadcaster.clearCache();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8882ad26/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 0d8d4d5..1feb66f 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -55,6 +55,7 @@ import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.helix.HelixJobEngineAdmin;
import org.apache.kylin.rest.request.MetricsRequest;
import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.response.MetricsResponse;
@@ -569,9 +570,10 @@ public class CubeService extends BasicService {
public void updateOnNewSegmentReady(String cubeName) {
logger.debug("on updateOnNewSegmentReady: " + cubeName);
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- String serverMode = kylinConfig.getServerMode();
- logger.debug("server mode: " + serverMode);
- if (Constant.SERVER_MODE_JOB.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase())) {
+ HelixJobEngineAdmin jobEngineAdmin = HelixJobEngineAdmin.getInstance(kylinConfig.getZookeeperAddress());
+ boolean isLeaderRole = jobEngineAdmin.isLeaderRole(kylinConfig.getClusterName(), HelixJobEngineAdmin.getCurrentInstanceName());
+ logger.debug("server is leader role ? " + isLeaderRole);
+ if (isLeaderRole == true) {
keepCubeRetention(cubeName);
mergeCubeSegment(cubeName);
}