You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/05/28 05:24:19 UTC
incubator-kylin git commit: KYLIN-783 use standard sandbox configs
rather than ambari populated configs
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 37f3b6344 -> 746d4adc5
KYLIN-783 use standard sandbox configs rather than ambari populated configs
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/746d4adc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/746d4adc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/746d4adc
Branch: refs/heads/0.8.0
Commit: 746d4adc58a176201757b23627d6a08dc28c2819
Parents: 37f3b63
Author: honma <ho...@ebay.com>
Authored: Thu May 28 11:22:30 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu May 28 11:23:31 2015 +0800
----------------------------------------------------------------------
.../sandbox/capacity-scheduler.xml | 11 ++-
examples/test_case_data/sandbox/core-site.xml | 72 ++++++++++++++++----
.../test_case_data/sandbox/hadoop-policy.xml | 2 +-
.../test_case_data/sandbox/hbase-policy.xml | 4 +-
examples/test_case_data/sandbox/hbase-site.xml | 15 ++--
examples/test_case_data/sandbox/hdfs-site.xml | 43 ++++++++++--
examples/test_case_data/sandbox/hive-site.xml | 46 ++++++-------
examples/test_case_data/sandbox/mapred-site.xml | 38 ++++++++---
examples/test_case_data/sandbox/yarn-site.xml | 55 ++++++++++++---
9 files changed, 212 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/746d4adc/examples/test_case_data/sandbox/capacity-scheduler.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/capacity-scheduler.xml b/examples/test_case_data/sandbox/capacity-scheduler.xml
index 9c97e43..dcd63c1 100644
--- a/examples/test_case_data/sandbox/capacity-scheduler.xml
+++ b/examples/test_case_data/sandbox/capacity-scheduler.xml
@@ -1,4 +1,4 @@
-<!--Mon May 25 01:30:55 2015-->
+<!--Wed May 27 08:09:06 2015-->
<configuration>
<property>
@@ -8,7 +8,7 @@
<property>
<name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
- <value>0.2</value>
+ <value>0.5</value>
</property>
<property>
@@ -53,7 +53,7 @@
<property>
<name>yarn.scheduler.capacity.root.default-node-label-expression</name>
- <value> </value>
+ <value></value>
</property>
<property>
@@ -72,6 +72,11 @@
</property>
<property>
+ <name>yarn.scheduler.capacity.root.default.maximum-am-resource-percent</name>
+ <value>0.5</value>
+ </property>
+
+ <property>
<name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
<value>100</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/746d4adc/examples/test_case_data/sandbox/core-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/core-site.xml b/examples/test_case_data/sandbox/core-site.xml
index e2b3fbe..0a80866 100644
--- a/examples/test_case_data/sandbox/core-site.xml
+++ b/examples/test_case_data/sandbox/core-site.xml
@@ -1,4 +1,4 @@
-<!--Mon May 25 01:30:55 2015-->
+<!--Wed May 27 08:08:58 2015-->
<configuration>
<property>
@@ -13,16 +13,41 @@
</property>
<property>
+ <name>ha.failover-controller.active-standby-elector.zk.op.retries</name>
+ <value>120</value>
+ </property>
+
+ <property>
<name>hadoop.http.authentication.simple.anonymous.allowed</name>
<value>true</value>
</property>
<property>
- <name>hadoop.proxyuser.hcat.groups</name>
+ <name>hadoop.proxyuser.falcon.groups</name>
+ <value>users</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.falcon.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hbase.groups</name>
<value>users</value>
</property>
<property>
+ <name>hadoop.proxyuser.hbase.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hcat.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
<name>hadoop.proxyuser.hcat.hosts</name>
<value>sandbox.hortonworks.com</value>
</property>
@@ -34,18 +59,42 @@
<property>
<name>hadoop.proxyuser.hive.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hue.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.hue.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.oozie.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.oozie.hosts</name>
<value>sandbox.hortonworks.com</value>
</property>
<property>
+ <name>hadoop.proxyuser.root.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.proxyuser.root.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
<name>hadoop.security.auth_to_local</name>
- <value>
- RULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/
- RULE:[2:$1@$0](jhs@.*)s/.*/mapred/
- RULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/
- RULE:[2:$1@$0](hm@.*)s/.*/hbase/
- RULE:[2:$1@$0](rs@.*)s/.*/hbase/
- DEFAULT</value>
+ <value>DEFAULT</value>
</property>
<property>
@@ -98,9 +147,4 @@
<value>false</value>
</property>
- <property>
- <name>proxyuser_group</name>
- <value>users</value>
- </property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/746d4adc/examples/test_case_data/sandbox/hadoop-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hadoop-policy.xml b/examples/test_case_data/sandbox/hadoop-policy.xml
index 3f08465..4556120 100644
--- a/examples/test_case_data/sandbox/hadoop-policy.xml
+++ b/examples/test_case_data/sandbox/hadoop-policy.xml
@@ -1,4 +1,4 @@
-<!--Mon May 25 01:30:15 2015-->
+<!--Fri Apr 24 12:58:40 2015-->
<configuration>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/746d4adc/examples/test_case_data/sandbox/hbase-policy.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-policy.xml b/examples/test_case_data/sandbox/hbase-policy.xml
index 6549239..ae974bb 100644
--- a/examples/test_case_data/sandbox/hbase-policy.xml
+++ b/examples/test_case_data/sandbox/hbase-policy.xml
@@ -1,4 +1,4 @@
-<!--Mon May 25 01:30:31 2015-->
+<!--Wed May 27 06:14:03 2015-->
<configuration>
<property>
@@ -16,4 +16,4 @@
<value>*</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/746d4adc/examples/test_case_data/sandbox/hbase-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hbase-site.xml b/examples/test_case_data/sandbox/hbase-site.xml
index b63cc90..06755a8 100644
--- a/examples/test_case_data/sandbox/hbase-site.xml
+++ b/examples/test_case_data/sandbox/hbase-site.xml
@@ -1,4 +1,4 @@
-<!--Mon May 25 01:30:31 2015-->
+<!--Wed May 27 06:14:00 2015-->
<configuration>
<property>
@@ -23,12 +23,12 @@
<property>
<name>hbase.coprocessor.master.classes</name>
- <value></value>
+ <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
</property>
<property>
<name>hbase.coprocessor.region.classes</name>
- <value></value>
+ <value>com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor</value>
</property>
<property>
@@ -122,13 +122,18 @@
</property>
<property>
+ <name>hbase.rpc.protection</name>
+ <value>PRIVACY</value>
+ </property>
+
+ <property>
<name>hbase.security.authentication</name>
<value>simple</value>
</property>
<property>
<name>hbase.security.authorization</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
@@ -171,4 +176,4 @@
<value>/hbase-unsecure</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/746d4adc/examples/test_case_data/sandbox/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hdfs-site.xml b/examples/test_case_data/sandbox/hdfs-site.xml
index c9a3a6a..db4da75 100644
--- a/examples/test_case_data/sandbox/hdfs-site.xml
+++ b/examples/test_case_data/sandbox/hdfs-site.xml
@@ -1,9 +1,14 @@
-<!--Mon May 25 01:30:31 2015-->
+<!--Wed May 27 06:14:02 2015-->
<configuration>
<property>
<name>dfs.block.access.token.enable</name>
- <value>true</value>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>dfs.block.size</name>
+ <value>34217472</value>
</property>
<property>
@@ -28,7 +33,7 @@
<property>
<name>dfs.cluster.administrators</name>
- <value> hdfs</value>
+ <value>hdfs</value>
</property>
<property>
@@ -44,6 +49,7 @@
<property>
<name>dfs.datanode.data.dir</name>
<value>/hadoop/hdfs/data</value>
+ <final>true</final>
</property>
<property>
@@ -59,6 +65,7 @@
<property>
<name>dfs.datanode.failed.volumes.tolerated</name>
<value>0</value>
+ <final>true</final>
</property>
<property>
@@ -78,7 +85,12 @@
<property>
<name>dfs.datanode.max.transfer.threads</name>
- <value>16384</value>
+ <value>1024</value>
+ </property>
+
+ <property>
+ <name>dfs.datanode.max.xcievers</name>
+ <value>1024</value>
</property>
<property>
@@ -117,8 +129,13 @@
</property>
<property>
+ <name>dfs.journalnode.https-address</name>
+ <value>0.0.0.0:8481</value>
+ </property>
+
+ <property>
<name>dfs.namenode.accesstime.precision</name>
- <value>0</value>
+ <value>3600000</value>
</property>
<property>
@@ -153,7 +170,7 @@
<property>
<name>dfs.namenode.handler.count</name>
- <value>40</value>
+ <value>100</value>
</property>
<property>
@@ -170,6 +187,7 @@
<property>
<name>dfs.namenode.name.dir</name>
<value>/hadoop/hdfs/namenode</value>
+ <final>true</final>
</property>
<property>
@@ -203,6 +221,16 @@
</property>
<property>
+ <name>dfs.nfs.exports.allowed.hosts</name>
+ <value>* rw</value>
+ </property>
+
+ <property>
+ <name>dfs.nfs3.dump.dir</name>
+ <value>/tmp/.hdfs-nfs</value>
+ </property>
+
+ <property>
<name>dfs.permissions.enabled</name>
<value>true</value>
</property>
@@ -214,7 +242,7 @@
<property>
<name>dfs.replication</name>
- <value>3</value>
+ <value>1</value>
</property>
<property>
@@ -231,6 +259,7 @@
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
+ <final>true</final>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/746d4adc/examples/test_case_data/sandbox/hive-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/hive-site.xml b/examples/test_case_data/sandbox/hive-site.xml
index 7efaf4f..f07a290 100644
--- a/examples/test_case_data/sandbox/hive-site.xml
+++ b/examples/test_case_data/sandbox/hive-site.xml
@@ -1,4 +1,4 @@
-<!--Mon May 25 01:23:17 2015-->
+<!--Fri Apr 24 13:44:42 2015-->
<configuration>
<property>
@@ -23,7 +23,7 @@
<property>
<name>hive.auto.convert.join.noconditionaltask.size</name>
- <value>1251999744</value>
+ <value>1000000000</value>
</property>
<property>
@@ -32,11 +32,6 @@
</property>
<property>
- <name>hive.auto.convert.sortmerge.join.noconditionaltask</name>
- <value>true</value>
- </property>
-
- <property>
<name>hive.auto.convert.sortmerge.join.to.mapjoin</name>
<value>false</value>
</property>
@@ -73,7 +68,7 @@
<property>
<name>hive.compactor.check.interval</name>
- <value>300L</value>
+ <value>300s</value>
</property>
<property>
@@ -88,7 +83,7 @@
<property>
<name>hive.compactor.initiator.on</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
@@ -98,7 +93,7 @@
<property>
<name>hive.compactor.worker.timeout</name>
- <value>86400L</value>
+ <value>86400s</value>
</property>
<property>
@@ -233,7 +228,7 @@
<property>
<name>hive.execution.engine</name>
- <value>mr</value>
+ <value>tez</value>
</property>
<property>
@@ -253,7 +248,7 @@
<property>
<name>hive.heapsize</name>
- <value>1024</value>
+ <value>250</value>
</property>
<property>
@@ -427,11 +422,6 @@
</property>
<property>
- <name>hive.optimize.mapjoin.mapreduce</name>
- <value>true</value>
- </property>
-
- <property>
<name>hive.optimize.metadataonly</name>
<value>true</value>
</property>
@@ -528,7 +518,7 @@
<property>
<name>hive.server2.enable.doAs</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -638,7 +628,7 @@
<property>
<name>hive.support.concurrency</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
@@ -648,7 +638,7 @@
<property>
<name>hive.tez.container.size</name>
- <value>3584</value>
+ <value>250</value>
</property>
<property>
@@ -678,7 +668,7 @@
<property>
<name>hive.tez.java.opts</name>
- <value>-server -Xmx2868m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps</value>
+ <value>-server -Xmx200m -Djava.net.preferIPv4Stack=true</value>
</property>
<property>
@@ -703,7 +693,7 @@
<property>
<name>hive.txn.manager</name>
- <value>org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager</value>
+ <value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
</property>
<property>
@@ -722,6 +712,11 @@
</property>
<property>
+ <name>hive.users.in.admin.role</name>
+ <value>hue,hive</value>
+ </property>
+
+ <property>
<name>hive.vectorized.execution.enabled</name>
<value>true</value>
</property>
@@ -762,6 +757,11 @@
</property>
<property>
+ <name>hive_metastore_user_passwd</name>
+ <value>hive</value>
+ </property>
+
+ <property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.jdbc.Driver</value>
</property>
@@ -776,4 +776,4 @@
<value>hive</value>
</property>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/746d4adc/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index 674643a..8d345d1 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -1,7 +1,27 @@
-<!--Mon May 25 01:30:55 2015-->
+<!--Wed May 27 08:09:05 2015-->
<configuration>
<property>
+ <name>io.sort.mb</name>
+ <value>64</value>
+ </property>
+
+ <property>
+ <name>mapred.child.java.opts</name>
+ <value>-Xmx200m</value>
+ </property>
+
+ <property>
+ <name>mapred.job.map.memory.mb</name>
+ <value>250</value>
+ </property>
+
+ <property>
+ <name>mapred.job.reduce.memory.mb</name>
+ <value>250</value>
+ </property>
+
+ <property>
<name>mapreduce.admin.map.child.java.opts</name>
<value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value>
</property>
@@ -33,7 +53,7 @@
<property>
<name>mapreduce.cluster.administrators</name>
- <value> hadoop</value>
+ <value>hadoop</value>
</property>
<property>
@@ -78,7 +98,7 @@
<property>
<name>mapreduce.map.java.opts</name>
- <value>-Xmx2867m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -88,7 +108,7 @@
<property>
<name>mapreduce.map.memory.mb</name>
- <value>3584</value>
+ <value>250</value>
</property>
<property>
@@ -123,7 +143,7 @@
<property>
<name>mapreduce.reduce.java.opts</name>
- <value>-Xmx2867m</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -133,7 +153,7 @@
<property>
<name>mapreduce.reduce.memory.mb</name>
- <value>3584</value>
+ <value>250</value>
</property>
<property>
@@ -183,7 +203,7 @@
<property>
<name>mapreduce.task.io.sort.mb</name>
- <value>1024</value>
+ <value>64</value>
</property>
<property>
@@ -198,7 +218,7 @@
<property>
<name>yarn.app.mapreduce.am.command-opts</name>
- <value>-Xmx2867m -Dhdp.version=${hdp.version}</value>
+ <value>-Xmx200m</value>
</property>
<property>
@@ -208,7 +228,7 @@
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
- <value>3584</value>
+ <value>250</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/746d4adc/examples/test_case_data/sandbox/yarn-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/yarn-site.xml b/examples/test_case_data/sandbox/yarn-site.xml
index f764f1b..a8cda86 100644
--- a/examples/test_case_data/sandbox/yarn-site.xml
+++ b/examples/test_case_data/sandbox/yarn-site.xml
@@ -1,4 +1,4 @@
-<!--Mon May 25 01:30:55 2015-->
+<!--Wed May 27 08:09:01 2015-->
<configuration>
<property>
@@ -28,7 +28,7 @@
<property>
<name>yarn.client.nodemanager-connect.max-wait-ms</name>
- <value>900000</value>
+ <value>60000</value>
</property>
<property>
@@ -37,6 +37,11 @@
</property>
<property>
+ <name>yarn.http.policy</name>
+ <value>HTTP_ONLY</value>
+ </property>
+
+ <property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
@@ -192,13 +197,18 @@
</property>
<property>
+ <name>yarn.nodemanager.pmem-check-enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
<name>yarn.nodemanager.recovery.dir</name>
<value>/var/log/hadoop-yarn/nodemanager/recovery-state</value>
</property>
<property>
<name>yarn.nodemanager.recovery.enabled</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
@@ -218,7 +228,7 @@
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
- <value>14336</value>
+ <value>9216</value>
</property>
<property>
@@ -233,7 +243,7 @@
<property>
<name>yarn.nodemanager.vmem-pmem-ratio</name>
- <value>2.1</value>
+ <value>10</value>
</property>
<property>
@@ -273,7 +283,7 @@
<property>
<name>yarn.resourcemanager.fs.state-store.uri</name>
- <value> </value>
+ <value></value>
</property>
<property>
@@ -293,7 +303,7 @@
<property>
<name>yarn.resourcemanager.recovery.enabled</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
@@ -342,8 +352,33 @@
</property>
<property>
+ <name>yarn.resourcemanager.webapp.https.address</name>
+ <value>localhost:8090</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.proxyuser.hcat.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.proxyuser.hcat.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.proxyuser.oozie.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.webapp.proxyuser.oozie.hosts</name>
+ <value>*</value>
+ </property>
+
+ <property>
<name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
@@ -383,12 +418,12 @@
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
- <value>14336</value>
+ <value>9216</value>
</property>
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
- <value>3584</value>
+ <value>1536</value>
</property>
<property>