You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/08/05 05:09:24 UTC
[2/2] incubator-kylin git commit: KYLIN-908 Help community developer
to setup develop/debug environment
KYLIN-908 Help community developer to setup develop/debug environment
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/aebd1917
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/aebd1917
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/aebd1917
Branch: refs/heads/0.8
Commit: aebd19175d1a9a06bd8e632ae102f3a71cb942c5
Parents: 35cfbe4
Author: honma <ho...@ebay.com>
Authored: Wed Jul 29 10:22:24 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Aug 5 11:06:59 2015 +0800
----------------------------------------------------------------------
bin/metastore.sh | 25 ++++++++
.../kylin/job/manager/ExecutableManager.java | 21 ++++---
.../java/org/apache/kylin/rest/DebugTomcat.java | 7 ++-
.../storage/hbase/SandboxMetastoreCLI.java | 65 ++++++++++++++++++++
4 files changed, 107 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/aebd1917/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/bin/metastore.sh b/bin/metastore.sh
old mode 100644
new mode 100755
index 33d22d2..4c021a3
--- a/bin/metastore.sh
+++ b/bin/metastore.sh
@@ -1,5 +1,30 @@
#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script is for production metadata store manipulation
+# It is desinged to run in hadoop CLI, both in sandbox or in real hadoop environment
+#
+# If you're a developper of Kylin and want to download sandbox's metadata into your dev machine,
+# take a look at SandboxMetastoreCLI
+
+
+
dir=$(dirname ${0})
source ${dir}/check-env.sh
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/aebd1917/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
index 65abf7b..ba8401a 100644
--- a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java
@@ -18,7 +18,6 @@
package org.apache.kylin.job.manager;
-import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -33,7 +32,6 @@ import org.apache.kylin.job.execution.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nullable;
import java.lang.reflect.Constructor;
import java.util.HashMap;
import java.util.List;
@@ -85,7 +83,7 @@ public class ExecutableManager {
executableOutputPO.setUuid(executable.getId());
executableDao.addJobOutput(executableOutputPO);
if (executable instanceof DefaultChainedExecutable) {
- for (AbstractExecutable subTask: ((DefaultChainedExecutable) executable).getTasks()) {
+ for (AbstractExecutable subTask : ((DefaultChainedExecutable) executable).getTasks()) {
addJobOutput(subTask);
}
}
@@ -146,13 +144,16 @@ public class ExecutableManager {
public List<AbstractExecutable> getAllExecutables() {
try {
- return Lists.transform(executableDao.getJobs(), new Function<ExecutablePO, AbstractExecutable>() {
- @Nullable
- @Override
- public AbstractExecutable apply(ExecutablePO input) {
- return parseTo(input);
+ List<AbstractExecutable> ret = Lists.newArrayList();
+ for (ExecutablePO po : executableDao.getJobs()) {
+ try {
+ AbstractExecutable ae = parseTo(po);
+ ret.add(ae);
+ } catch (IllegalArgumentException e) {
+ logger.error("error parsing one executabePO: ", e);
}
- });
+ }
+ return ret;
} catch (PersistentException e) {
logger.error("error get All Jobs", e);
throw new RuntimeException(e);
@@ -305,7 +306,7 @@ public class ExecutableManager {
List<ExecutablePO> tasks = executablePO.getTasks();
if (tasks != null && !tasks.isEmpty()) {
Preconditions.checkArgument(result instanceof ChainedExecutable);
- for (ExecutablePO subTask: tasks) {
+ for (ExecutablePO subTask : tasks) {
((ChainedExecutable) result).addTask(parseTo(subTask));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/aebd1917/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index 75c2de7..e285602 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -41,6 +41,11 @@ public class DebugTomcat {
overrideDevJobJarLocations();
System.setProperty("spring.profiles.active", "testing");
+
+ //avoid log permission issue
+ if (System.getProperty("catalina.home") == null)
+ System.setProperty("catalina.home", ".");
+
if (System.getProperty("hdp.version") == null)
System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
@@ -101,7 +106,7 @@ public class DebugTomcat {
String webBase = new File("../webapp/app").getAbsolutePath();
if (new File(webBase, "WEB-INF").exists() == false) {
- throw new RuntimeException("In order to launch Kylin web app from IDE, please make a symblink from webapp/app/WEB-INF to server/src/main/webapp/WEB-INF");
+ throw new RuntimeException("In order to launch Kylin web app from IDE, please copy server/src/main/webapp/WEB-INF to webapp/app/WEB-INF");
}
Tomcat tomcat = new Tomcat();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/aebd1917/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/SandboxMetastoreCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/SandboxMetastoreCLI.java
new file mode 100644
index 0000000..aee065c
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/SandboxMetastoreCLI.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package org.apache.kylin.storage.hbase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceTool;
+import org.apache.kylin.common.util.ClassUtil;
+
+import java.io.File;
+
+/**
+ * This is a helper class for developer to directly manipulate the metadata store in sandbox
+ * This is designed to run in IDE(i.e. not on sandbox hadoop CLI)
+ *
+ * For production metadata store manipulation refer to bin/metastore.sh in binary package
+ * It is desinged to run in hadoop CLI, both in sandbox or in real hadoop environment
+ */
+public class SandboxMetastoreCLI {
+
+ private static final Log logger = LogFactory.getLog(SandboxMetastoreCLI.class);
+
+ public static void main(String[] args) throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
+ System.setProperty("hdp.version", "2.2.4.2-2"); // mapred-site.xml ref this
+
+ if (args.length < 2) {
+ printUsage();
+ return;
+ }
+
+ if ("download".equalsIgnoreCase(args[0])) {
+ ResourceTool.main(new String[] { "download", args[1] });
+ } else if ("upload".equalsIgnoreCase(args[0])) {
+ ResourceTool.main(new String[] { "upload", args[1] });
+ } else {
+ printUsage();
+ }
+ }
+
+ private static void printUsage() {
+ logger.info("Usage: SandboxMetastoreCLI download toFolder");
+ logger.info("Usage: SandboxMetastoreCLI upload fromFolder");
+ }
+}