You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/01/23 09:32:18 UTC
[31/50] [abbrv] incubator-kylin git commit: KYLIN-564 leave lzo
enableness totally a user option
KYLIN-564 leave lzo enableness totally a user option
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/befea542
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/befea542
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/befea542
Branch: refs/heads/inverted-index
Commit: befea542dc40843376b3d447db68ad5a158a429d
Parents: 2867fcc
Author: honma <ho...@ebay.com>
Authored: Thu Jan 22 21:51:16 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu Jan 22 21:51:16 2015 +0800
----------------------------------------------------------------------
.../sandbox/kylin_job_conf.lzo_disabled.xml | 60 --------------------
.../sandbox/kylin_job_conf.lzo_enabled.xml | 60 --------------------
.../job/tools/LZOSupportnessChecker.java | 4 ++
.../kylinolap/job/BuildCubeWithEngineTest.java | 1 -
.../test/java/com/kylinolap/job/DeployUtil.java | 11 ----
.../com/kylinolap/job/SampleCubeSetupTest.java | 4 --
kylin.sh | 7 ++-
7 files changed, 10 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/befea542/examples/test_case_data/sandbox/kylin_job_conf.lzo_disabled.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.lzo_disabled.xml b/examples/test_case_data/sandbox/kylin_job_conf.lzo_disabled.xml
deleted file mode 100644
index f62ce55..0000000
--- a/examples/test_case_data/sandbox/kylin_job_conf.lzo_disabled.xml
+++ /dev/null
@@ -1,60 +0,0 @@
-<?xml version="1.0"?>
-<configuration>
-
- <property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
- <value>-1</value>
- <description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value. No limits if set to -1.
- </description>
- </property>
-
- <!-- uncomment the following 5 properties to enable lzo compressing
-
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
-
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
-
- <property>
- <name>mapred.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
-
- !-->
-
- <property>
- <name>mapreduce.job.max.split.locations</name>
- <value>2000</value>
- <description>No description</description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>2</value>
- <description>Block replication</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/befea542/examples/test_case_data/sandbox/kylin_job_conf.lzo_enabled.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_job_conf.lzo_enabled.xml b/examples/test_case_data/sandbox/kylin_job_conf.lzo_enabled.xml
deleted file mode 100644
index f62ce55..0000000
--- a/examples/test_case_data/sandbox/kylin_job_conf.lzo_enabled.xml
+++ /dev/null
@@ -1,60 +0,0 @@
-<?xml version="1.0"?>
-<configuration>
-
- <property>
- <name>mapreduce.job.split.metainfo.maxsize</name>
- <value>-1</value>
- <description>The maximum permissible size of the split metainfo file.
- The JobTracker won't attempt to read split metainfo files bigger than
- the configured value. No limits if set to -1.
- </description>
- </property>
-
- <!-- uncomment the following 5 properties to enable lzo compressing
-
- <property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
- <description>Compress map outputs</description>
- </property>
-
- <property>
- <name>mapred.map.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for map outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compress</name>
- <value>true</value>
- <description>Compress the output of a MapReduce job</description>
- </property>
-
- <property>
- <name>mapred.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
- <description>The compression codec to use for job outputs
- </description>
- </property>
-
- <property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
- <description>The compression type to use for job outputs</description>
- </property>
-
- !-->
-
- <property>
- <name>mapreduce.job.max.split.locations</name>
- <value>2000</value>
- <description>No description</description>
- </property>
-
- <property>
- <name>dfs.replication</name>
- <value>2</value>
- <description>Block replication</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/befea542/job/src/main/java/com/kylinolap/job/tools/LZOSupportnessChecker.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/com/kylinolap/job/tools/LZOSupportnessChecker.java b/job/src/main/java/com/kylinolap/job/tools/LZOSupportnessChecker.java
index 62dc594..10babf6 100644
--- a/job/src/main/java/com/kylinolap/job/tools/LZOSupportnessChecker.java
+++ b/job/src/main/java/com/kylinolap/job/tools/LZOSupportnessChecker.java
@@ -18,4 +18,8 @@ public class LZOSupportnessChecker {
}
return true;
}
+
+ public static void main(String[] args) throws Exception {
+ System.out.println("LZO supported by current env? " + getSupportness());
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/befea542/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java
index dc56ad8..2d1ae5e 100644
--- a/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/com/kylinolap/job/BuildCubeWithEngineTest.java
@@ -65,7 +65,6 @@ public class BuildCubeWithEngineTest extends HBaseMetadataTestCase {
DeployUtil.initCliWorkDir();
DeployUtil.deployMetadata();
DeployUtil.overrideJobJarLocations();
- DeployUtil.overrideJobConf(SANDBOX_TEST_DATA);
engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
jobManager = new JobManager("Build_Test_Cube_Engine", engineConfig);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/befea542/job/src/test/java/com/kylinolap/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/DeployUtil.java b/job/src/test/java/com/kylinolap/job/DeployUtil.java
index cebe1f0..ecedfda 100644
--- a/job/src/test/java/com/kylinolap/job/DeployUtil.java
+++ b/job/src/test/java/com/kylinolap/job/DeployUtil.java
@@ -95,17 +95,6 @@ public class DeployUtil {
return new Pair<File, File>(jobJar, coprocessorJar);
}
- public static void overrideJobConf(String confDir) throws IOException {
- boolean enableLzo = LZOSupportnessChecker.getSupportness();
- overrideJobConf(confDir, enableLzo);
- }
-
- public static void overrideJobConf(String confDir, boolean enableLzo) throws IOException {
- File src = new File(confDir, JobEngineConfig.HADOOP_JOB_CONF_FILENAME + (enableLzo ? ".lzo_enabled" : ".lzo_disabled") + ".xml");
- File dst = new File(confDir, JobEngineConfig.HADOOP_JOB_CONF_FILENAME + ".xml");
- FileUtils.copyFile(src, dst);
- }
-
private static void execCliCommand(String cmd) throws IOException {
config().getCliCommandExecutor().execute(cmd);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/befea542/job/src/test/java/com/kylinolap/job/SampleCubeSetupTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/com/kylinolap/job/SampleCubeSetupTest.java b/job/src/test/java/com/kylinolap/job/SampleCubeSetupTest.java
index 30ed697..92d3bb1 100644
--- a/job/src/test/java/com/kylinolap/job/SampleCubeSetupTest.java
+++ b/job/src/test/java/com/kylinolap/job/SampleCubeSetupTest.java
@@ -72,10 +72,6 @@ public class SampleCubeSetupTest extends HBaseMetadataTestCase {
}
private void deployJobConfToEtc() throws IOException {
- String lzoSupportness = System.getenv("KYLIN_LZO_SUPPORTED");
- boolean enableLzo = "true".equalsIgnoreCase(lzoSupportness);
- DeployUtil.overrideJobConf(SANDBOX_TEST_DATA, enableLzo);
-
File src = new File(SANDBOX_TEST_DATA, JobEngineConfig.HADOOP_JOB_CONF_FILENAME + ".xml");
File dst = new File("/etc/kylin", src.getName());
FileUtils.copyFile(src, dst);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/befea542/kylin.sh
----------------------------------------------------------------------
diff --git a/kylin.sh b/kylin.sh
index d04285a..2669da5 100755
--- a/kylin.sh
+++ b/kylin.sh
@@ -8,6 +8,11 @@ else
echo "CATALINA_HOME is set to $CATALINA_HOME"
fi
+if [ ! -z "$KYLIN_LD_LIBRARY_PATH" ]
+then
+ echo "KYLIN_LD_LIBRARY_PATH is set to $KYLIN_LD_LIBRARY_PATH"
+fi
+
#The location of all hadoop/hbase configurations are difficult to get.
#Plus, some of the system properties are secretly set in hadoop/hbase shell command.
#For example, in hdp 2.2, there is a system property called hdp.version,
@@ -18,7 +23,7 @@ fi
#hbase command will do all the dirty tasks for us:
if [ "$1" = "start" ] ; then
export HBASE_CLASSPATH_PREFIX=/etc/kylin:${CATALINA_HOME}/bin/bootstrap.jar:${CATALINA_HOME}/bin/tomcat-juli.jar:${CATALINA_HOME}/lib/*:$HBASE_CLASSPATH_PREFIX
- hbase -Djava.util.logging.config.file=${CATALINA_HOME}/conf/logging.properties -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true -Dspring.profiles.active=sandbox -Djava.endorsed.dirs=${CATALINA_HOME}/endorsed -Dcatalina.base=${CATALINA_HOME} -Dcatalina.home=${CATALINA_HOME} -Djava.io.tmpdir=${CATALINA_HOME}/temp org.apache.hadoop.util.RunJar ${CATALINA_HOME}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${CATALINA_HOME}/logs/kylin_sandbox.log 2>&1 &
+ hbase -Djava.util.logging.config.file=${CATALINA_HOME}/conf/logging.properties -Djava.library.path=${KYLIN_LD_LIBRARY_PATH} -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true -Dspring.profiles.active=sandbox -Djava.endorsed.dirs=${CATALINA_HOME}/endorsed -Dcatalina.base=${CATALINA_HOME} -Dcatalina.home=${CATALINA_HOME} -Djava.io.tmpdir=${CATALINA_HOME}/temp org.apache.hadoop.util.RunJar ${CATALINA_HOME}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${CATALINA_HOME}/logs/kylin_sandbox.log 2>&1 &
echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\""
echo "Please visit http://<your_sandbox_ip>:7070/kylin to play with the cubes! (Useranme: ADMIN, Password: KYLIN)"
echo "You can check the log at ${CATALINA_HOME}/logs/kylin_sandbox.log"