You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/03/16 14:09:51 UTC
kylin git commit: KYLIN-1501 Add some API for runnable tasks on init
Repository: kylin
Updated Branches:
refs/heads/master 2f31d30f7 -> f1ecd8ef9
KYLIN-1501 Add some API for runnable tasks on init
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f1ecd8ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f1ecd8ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f1ecd8ef
Branch: refs/heads/master
Commit: f1ecd8ef9d4fd9fc698a44c7c1a02e95e78f6bae
Parents: 2f31d30
Author: lidongsjtu <li...@apache.org>
Authored: Wed Mar 16 21:02:32 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Wed Mar 16 21:08:29 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 15 ++---
.../org/apache/kylin/rest/init/InitialTask.java | 26 +++++++++
.../kylin/rest/init/InitialTaskManager.java | 58 ++++++++++++++++++++
.../src/main/resources/applicationContext.xml | 2 +
4 files changed, 94 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f1ecd8ef/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4ee1947..82f5434 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -122,7 +122,7 @@ public class KylinConfigBase implements Serializable {
public boolean isDevEnv() {
return "DEV".equals(getOptional("deploy.env", "DEV"));
}
-
+
public String getMetadataUrl() {
return getOptional("kylin.metadata.url");
}
@@ -179,10 +179,7 @@ public class KylinConfigBase implements Serializable {
if (!root.endsWith("/")) {
root += "/";
}
- return new StringBuffer(root)
- .append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-'))
- .append("/")
- .toString();
+ return new StringBuffer(root).append(StringUtils.replaceChars(getMetadataUrlPrefix(), ':', '-')).append("/").toString();
}
public CliCommandExecutor getCliCommandExecutor() throws IOException {
@@ -487,12 +484,12 @@ public class KylinConfigBase implements Serializable {
percent = Math.min(percent, 100);
return percent;
}
-
+
public Map<String, String> getCubingInMemMRJobConfOverride() {
// in-mem cubing requires big memory, however dev env (sandbox) may not have that much
String defaultOverride = isDevEnv() ? "" : "mapreduce.map.java.opts=-Xmx2700m; mapreduce.map.memory.mb=3072; mapreduce.task.io.sort.mb=200";
String override = getOptional("kylin.job.cubing.inmem.mrjob_conf_override", defaultOverride);
-
+
Map<String, String> result = Maps.newHashMap();
for (String pair : override.split(";")) {
int cut = pair.indexOf('=');
@@ -583,4 +580,8 @@ public class KylinConfigBase implements Serializable {
return getOptional("deploy.env", "DEV");
}
+ public String getInitTasks() {
+ return getOptional("kylin.init.tasks");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f1ecd8ef/server/src/main/java/org/apache/kylin/rest/init/InitialTask.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/init/InitialTask.java b/server/src/main/java/org/apache/kylin/rest/init/InitialTask.java
new file mode 100644
index 0000000..24691b1
--- /dev/null
+++ b/server/src/main/java/org/apache/kylin/rest/init/InitialTask.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.init;
+
+/**
+ * Created by dongli on 3/16/16.
+ */
+public abstract class InitialTask {
+ public abstract void execute();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f1ecd8ef/server/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java b/server/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
new file mode 100644
index 0000000..8912968
--- /dev/null
+++ b/server/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.init;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
+
+/**
+ * Created by dongli on 3/16/16.
+ */
+public class InitialTaskManager implements InitializingBean {
+
+ private static final Logger logger = LoggerFactory.getLogger(InitialTaskManager.class);
+
+ @Override
+ public void afterPropertiesSet() throws Exception {
+ logger.info("Kylin service is starting.....");
+
+ runInitialTasks();
+ }
+
+ private void runInitialTasks() {
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ String initTasks = kylinConfig.getInitTasks();
+ if (!StringUtils.isEmpty(initTasks)) {
+ String[] taskClasses = initTasks.split(",");
+ for (String taskClass : taskClasses) {
+ try {
+ InitialTask task = (InitialTask) Class.forName(taskClass).newInstance();
+ logger.info("Running task: " + taskClass);
+ task.execute();
+ } catch (Exception e) {
+ logger.error("Initial task failed: " + taskClass, e);
+ }
+ }
+ logger.info("All initial tasks finished.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f1ecd8ef/server/src/main/resources/applicationContext.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/applicationContext.xml b/server/src/main/resources/applicationContext.xml
index d7a93b9..db5b11c 100644
--- a/server/src/main/resources/applicationContext.xml
+++ b/server/src/main/resources/applicationContext.xml
@@ -38,6 +38,8 @@
<mvc:annotation-driven/>
<aop:aspectj-autoproxy/>
+ <bean class="org.apache.kylin.rest.init.InitialTaskManager" />
+
<context:component-scan base-package="org.apache.kylin.rest"/>
<bean class="org.apache.kylin.rest.security.PasswordPlaceholderConfigurer">