You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2013/05/01 23:52:22 UTC
[1/6] git commit: BIGTOP-839. add packaging test for Sqoop 2
Updated Branches:
refs/heads/master 3f03f6248 -> 39af29da4
BIGTOP-839. add packaging test for Sqoop 2
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/8292d06b
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/8292d06b
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/8292d06b
Branch: refs/heads/master
Commit: 8292d06ba5904dae9ece25baef5d51ca070d0892
Parents: 786706b
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Wed May 1 10:34:51 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Wed May 1 10:35:06 2013 -0700
----------------------------------------------------------------------
.../src/main/resources/yum/package_data.xml | 2 +-
.../src/main/resources/zypper/package_data.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8292d06b/bigtop-tests/test-artifacts/package/src/main/resources/yum/package_data.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/package/src/main/resources/yum/package_data.xml b/bigtop-tests/test-artifacts/package/src/main/resources/yum/package_data.xml
index 2771f2f..4d0ac42 100644
--- a/bigtop-tests/test-artifacts/package/src/main/resources/yum/package_data.xml
+++ b/bigtop-tests/test-artifacts/package/src/main/resources/yum/package_data.xml
@@ -143,7 +143,7 @@
<users>
<sqoop>
<home>/var/run/sqoop</home>
- <descr>Sqoop</descr>
+ <descr>Sqoop User</descr>
<shell>/sbin/nologin</shell>
</sqoop>
</users>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/8292d06b/bigtop-tests/test-artifacts/package/src/main/resources/zypper/package_data.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/package/src/main/resources/zypper/package_data.xml b/bigtop-tests/test-artifacts/package/src/main/resources/zypper/package_data.xml
index 57fc531..fb5fafc 100644
--- a/bigtop-tests/test-artifacts/package/src/main/resources/zypper/package_data.xml
+++ b/bigtop-tests/test-artifacts/package/src/main/resources/zypper/package_data.xml
@@ -140,7 +140,7 @@
<users>
<sqoop>
<home>/var/run/sqoop</home>
- <descr>Sqoop</descr>
+ <descr>Sqoop User</descr>
<shell>/sbin/nologin</shell>
</sqoop>
</users>
[4/6] git commit: BIGTOP-956. fix small issues with mahout,
hue and oozie tests
Posted by rv...@apache.org.
BIGTOP-956. fix small issues with mahout, hue and oozie tests
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a4eb310b
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a4eb310b
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a4eb310b
Branch: refs/heads/master
Commit: a4eb310b236314379aed4a82036d01fa8b51a992
Parents: 3414748
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Wed May 1 11:05:51 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Wed May 1 14:50:38 2013 -0700
----------------------------------------------------------------------
.../bigtop/itest/huesmoke/TestHueSmoke.groovy | 4 +-
.../itest/mahout/smoke/TestMahoutExamples.groovy | 69 ++++++++-------
.../bigtop/itest/ooziesmoke/TestOozieSmoke.groovy | 10 --
bigtop-tests/test-execution/smokes/hue/pom.xml | 34 +++++++-
4 files changed, 72 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy b/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy
index da12d50..50c0a46 100644
--- a/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy
+++ b/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.bigtop.itest.giraph.smoke
+package org.apache.bigtop.itest.hue.smoke
import static org.junit.Assert.assertEquals
import static org.junit.Assert.assertTrue
@@ -25,7 +25,7 @@ import org.apache.bigtop.itest.shell.Shell
import org.junit.Test
public class TestHueSmoke {
- final static String hueServer = "http://localhost:8888"
+ final static String hueServer = System.getProperty("org.apache.bigtop.itest.hue_url", "http://localhost:8888");
final static String loginURL = "${hueServer}/accounts/login/";
final static String checkURL = "${hueServer}/debug/check_config";
final static String creds = "username=admin&password=admin";
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy b/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy
index 586792c..8cc8efc 100644
--- a/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy
+++ b/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy
@@ -102,8 +102,12 @@ public class TestMahoutExamples {
}
}
- @Test(timeout=1200000L)
+ @Test(timeout=12000000L)
public void factorizeMovieLensRatings() {
+ // convert ratings
+ sh.exec("cat ${TEMP_DIR}/movielens/ml-1m/ratings.dat |sed -e s/::/,/g| cut -d, -f1,2,3 > ${TEMP_DIR}/movielens/ratings.csv");
+ assertEquals("Unexpected error from converting ratings", 0, sh.getRet());
+
// put ratings in hdfs
sh.exec("hadoop fs -mkdir ${WORK_DIR}/movielens",
"hadoop fs -put ${TEMP_DIR}/movielens/ratings.csv ${WORK_DIR}/movielens/ratings.csv");
@@ -218,40 +222,43 @@ public class TestMahoutExamples {
@Test(timeout=9000000L)
public void testBayesNewsgroupClassifier() {
- sh.exec("""mahout org.apache.mahout.classifier.bayes.PrepareTwentyNewsgroups \
- -p ${TEMP_DIR}/20news-bydate/20news-bydate-train \
- -o ${TEMP_DIR}/20news-bydate/bayes-train-input \
- -a org.apache.mahout.vectorizer.DefaultAnalyzer \
- -c UTF-8""");
+ // put bayes-train-input and bayes-test-input in hdfs
+ sh.exec("hadoop fs -mkdir ${WORK_DIR}/20news-vectors");
+ sh.exec("hadoop fs -put ${TEMP_DIR}/20news-all ${WORK_DIR}/20news-all");
+ assertEquals("Unable to put bayes-train-input in hdfs", 0, sh.getRet());
+
+ sh.exec("mahout seqdirectory -i ${WORK_DIR}/20news-all -o ${WORK_DIR}/20news-seq");
assertEquals("Unexpected error from running mahout", 0, sh.getRet());
- sh.exec("""mahout org.apache.mahout.classifier.bayes.PrepareTwentyNewsgroups \
- -p ${TEMP_DIR}/20news-bydate/20news-bydate-test \
- -o ${TEMP_DIR}/20news-bydate/bayes-test-input \
- -a org.apache.mahout.vectorizer.DefaultAnalyzer \
- -c UTF-8""");
+
+ sh.exec("mahout seq2sparse -i ${WORK_DIR}/20news-seq -o ${WORK_DIR}/20news-vectors -lnorm -nv -wt tfidf");
assertEquals("Unexpected error from running mahout", 0, sh.getRet());
- // put bayes-train-input and bayes-test-input in hdfs
- sh.exec("hadoop fs -mkdir ${WORK_DIR}/20news-bydate");
- sh.exec("hadoop fs -put ${TEMP_DIR}/20news-bydate/bayes-train-input ${WORK_DIR}/20news-bydate/bayes-train-input");
- assertEquals("Unable to put bayes-train-input in hdfs", 0, sh.getRet());
- sh.exec("hadoop fs -put ${TEMP_DIR}/20news-bydate/bayes-test-input ${WORK_DIR}/20news-bydate/bayes-test-input");
- assertEquals("Unable to put bayes-test-input in hdfs", 0, sh.getRet());
-
- sh.exec("""mahout trainclassifier \
--i ${WORK_DIR}/20news-bydate/bayes-train-input \
--o ${WORK_DIR}/20news-bydate/bayes-model \
--type bayes \
--ng 1 \
--source hdfs""");
+ sh.exec("""mahout split \
+-i ${WORK_DIR}/20news-vectors/tfidf-vectors \
+--trainingOutput ${WORK_DIR}/20news-train-vectors \
+--testOutput ${WORK_DIR}/20news-test-vectors \
+--randomSelectionPct 40 --overwrite --sequenceFiles -xm sequential""");
+ assertEquals("Unexpected error from running mahout", 0, sh.getRet());
+
+ sh.exec("""mahout trainnb \
+-i ${WORK_DIR}/20news-train-vectors -el \
+-o ${WORK_DIR}/model \
+-li ${WORK_DIR}/labelindex \
+-ow""");
assertEquals("Unexpected error from running mahout", 0, sh.getRet());
- sh.exec("""mahout testclassifier \
--m ${WORK_DIR}/20news-bydate/bayes-model \
--d ${WORK_DIR}/20news-bydate/bayes-test-input \
--type bayes \
--ng 1 \
--source hdfs \
--method mapreduce""");
+
+ sh.exec("""mahout testnb \
+-i ${WORK_DIR}/20news-train-vectors \
+-m ${WORK_DIR}/model \
+-l ${WORK_DIR}/labelindex \
+-ow -o ${WORK_DIR}/20news-testing""");
+ assertEquals("Unexpected error from running mahout", 0, sh.getRet());
+
+ sh.exec("""mahout testnb \
+-i ${WORK_DIR}/20news-test-vectors \
+-m ${WORK_DIR}/model \
+-l ${WORK_DIR}/labelindex \
+-ow -o ${WORK_DIR}/20news-testing""");
assertEquals("Unexpected error from running mahout", 0, sh.getRet());
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy b/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy
index 23a00cb..7d1a90c 100644
--- a/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy
+++ b/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy
@@ -125,16 +125,6 @@ class TestOozieSmoke {
}
@Test(timeout=300000L)
- public void testSqoop() {
- testOozieExamplesCommon("sqoop");
- }
-
- @Test(timeout=300000L)
- public void testSqoopFreeform() {
- testOozieExamplesCommon("sqoop-freeform");
- }
-
- @Test(timeout=300000L)
public void testSubwf() {
testOozieExamplesCommon("subwf");
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-execution/smokes/hue/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-execution/smokes/hue/pom.xml b/bigtop-tests/test-execution/smokes/hue/pom.xml
index ed96ae7..19e27a6 100644
--- a/bigtop-tests/test-execution/smokes/hue/pom.xml
+++ b/bigtop-tests/test-execution/smokes/hue/pom.xml
@@ -30,9 +30,11 @@
<groupId>org.apache.bigtop.itest</groupId>
<artifactId>hue-smoke-execution</artifactId>
<version>0.6.0-SNAPSHOT</version>
- <name>Giraph smoke test execution</name>
+ <name>Hue smoke test execution</name>
<properties>
+ <HUE_URL>${env.HUE_URL}</HUE_URL>
+
<org.apache.maven-dependency-plugin.groupId>org.apache.bigtop.itest</org.apache.maven-dependency-plugin.groupId>
<org.apache.maven-dependency-plugin.artifactId>hue-smoke</org.apache.maven-dependency-plugin.artifactId>
<org.apache.maven-dependency-plugin.version>${hue-smoke.version}</org.apache.maven-dependency-plugin.version>
@@ -55,8 +57,36 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <systemPropertyVariables>
+ <org.apache.bigtop.itest.hue_url>${HUE_URL}</org.apache.bigtop.itest.hue_url>
+ </systemPropertyVariables>
+ <forkMode>always</forkMode>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>enforce-property-oozie</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireProperty>
+ <property>HUE_URL</property>
+ <message>HUE_URL env. variable has to be set</message>
+ </requireProperty>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
</plugins>
</build>
-
</project>
[3/6] git commit: BIGTOP-951: Hive Smoke's TestJdbcDriver fails
Posted by rv...@apache.org.
BIGTOP-951: Hive Smoke's TestJdbcDriver fails
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/34147485
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/34147485
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/34147485
Branch: refs/heads/master
Commit: 341474850b8f76b3bfcb6a48493fcac06333fa40
Parents: 8292d06
Author: Mark Grover <gr...@gmail.com>
Authored: Tue Apr 30 18:20:44 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Wed May 1 10:51:55 2013 -0700
----------------------------------------------------------------------
.../bigtop/itest/hivesmoke/TestJdbcDriver.java | 2 +-
bigtop-tests/test-execution/smokes/hive/pom.xml | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/34147485/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/TestJdbcDriver.java b/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/TestJdbcDriver.java
index 44a921a..5a71a14 100644
--- a/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/TestJdbcDriver.java
+++ b/bigtop-tests/test-artifacts/hive/src/main/groovy/org/apache/bigtop/itest/hivesmoke/TestJdbcDriver.java
@@ -43,7 +43,7 @@ import org.apache.bigtop.itest.shell.Shell;
@Contract(
properties = {
- @Property(name="hiveserver.startup.wait", type=Property.Type.INT, longValue=1000, intValue=1000, defaultValue="1000")
+ @Property(name="hiveserver.startup.wait", type=Property.Type.INT, longValue=3000, intValue=3000, defaultValue="3000")
},
env = {})
public class TestJdbcDriver {
http://git-wip-us.apache.org/repos/asf/bigtop/blob/34147485/bigtop-tests/test-execution/smokes/hive/pom.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-execution/smokes/hive/pom.xml b/bigtop-tests/test-execution/smokes/hive/pom.xml
index 84f3f88..89e1036 100644
--- a/bigtop-tests/test-execution/smokes/hive/pom.xml
+++ b/bigtop-tests/test-execution/smokes/hive/pom.xml
@@ -108,7 +108,7 @@
<additionalClasspathElement>${HIVE_HOME}/lib/${hive-jdbc.jar}</additionalClasspathElement>
<additionalClasspathElement>${HIVE_HOME}/lib/${hive-metastore.jar}</additionalClasspathElement>
<additionalClasspathElement>${HIVE_HOME}/lib/${hive-service.jar}</additionalClasspathElement>
- <additionalClasspathElement>${HIVE_HOME}/lib/libfb303.jar</additionalClasspathElement>
+ <additionalClasspathElement>${HIVE_HOME}/lib/${libfb303.jar}</additionalClasspathElement>
<additionalClasspathElement>${HIVE_HOME}/lib/${slf4j-api.jar}</additionalClasspathElement>
</additionalClasspathElements>
</configuration>
@@ -127,7 +127,7 @@
<configuration>
<source>
if (!System.getProperties().keySet().contains('enforcer.skip')) {
- [ 'hive-hbase-handler.jar', 'hive-exec.jar', 'hive-jdbc.jar', 'hive-metastore.jar', 'hive-service.jar', 'slf4j-api.jar', 'guava.jar' ].each {
+ [ 'hive-hbase-handler.jar', 'hive-exec.jar', 'hive-jdbc.jar', 'hive-metastore.jar', 'hive-service.jar', 'libfb303.jar', 'slf4j-api.jar', 'guava.jar' ].each {
project.properties[it] =
org.apache.bigtop.itest.JarContent.getJarName(
"${HIVE_HOME}/lib", "${it.replaceAll(/.jar$/, '')}-.*\\.jar");
[5/6] git commit: BIGTOP-957. update deployment code for Sqoop2
Posted by rv...@apache.org.
BIGTOP-957. update deployment code for Sqoop2
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a36d79ef
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a36d79ef
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a36d79ef
Branch: refs/heads/master
Commit: a36d79efea604fb3a091ace8e888fcd6531d85d9
Parents: a4eb310
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Wed May 1 11:27:53 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Wed May 1 14:52:08 2013 -0700
----------------------------------------------------------------------
bigtop-deploy/puppet/manifests/cluster.pp | 7 +++++++
.../puppet/modules/hadoop-sqoop/manifests/init.pp | 10 +++++-----
.../puppet/modules/hadoop-sqoop/tests/init.pp | 4 ++--
3 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a36d79ef/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index 97b975b..605c6a9 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -229,6 +229,12 @@ class hadoop_head_node inherits hadoop_worker_node {
kerberos_realm => $kerberos_realm,
}
+
+ hadoop-sqoop::server { "sqoop server":
+ }
+ hadoop-sqoop::client { "sqoop client":
+ }
+
hue::server { "hue server":
rm_url => $hadoop_rm_url,
rm_proxy_url => $hadoop_rm_proxy_url,
@@ -241,6 +247,7 @@ class hadoop_head_node inherits hadoop_worker_node {
kerberos_realm => $kerberos_realm,
}
Hadoop::Httpfs<||> -> Hue::Server<||>
+ Hadoop-sqoop::Client<||> -> Hue::Server<||>
hadoop-zookeeper::server { "zookeeper":
myid => "0",
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a36d79ef/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp
index be69572..d1d08db 100644
--- a/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp
@@ -15,19 +15,19 @@
class hadoop-sqoop {
define client {
- package { "sqoop":
+ package { "sqoop-client":
ensure => latest,
}
}
- define metastore {
- package { "sqoop-metastore":
+ define server {
+ package { "sqoop-server":
ensure => latest,
}
- service { "sqoop-metastore":
+ service { "sqoop-server":
ensure => running,
- require => Package["sqoop-metastore"],
+ require => Package["sqoop-server"],
hasstatus => true,
hasrestart => true,
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a36d79ef/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp
index 336e80e..442323e 100644
--- a/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp
@@ -13,5 +13,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-hadoop-sqoop::client { "test-sqoop": }
-hadoop-sqoop::metastore { "test-sqoop-metastore": }
+hadoop-sqoop::client { "test-sqoop-client": }
+hadoop-sqoop::server { "test-sqoop-server": }
[2/6] git commit: BIGTOP-954. metadada and user data for sqoop and
hcatalog packages are wrong
Posted by rv...@apache.org.
BIGTOP-954. metadada and user data for sqoop and hcatalog packages are wrong
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/786706ba
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/786706ba
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/786706ba
Branch: refs/heads/master
Commit: 786706ba7b6ff0106077ec48db7c4215f61a8a9b
Parents: 3f03f62
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Tue Apr 30 17:38:15 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Wed May 1 10:35:06 2013 -0700
----------------------------------------------------------------------
bigtop-packages/src/deb/hcatalog/hcatalog.preinst | 2 +-
.../src/rpm/hcatalog/SPECS/hcatalog.spec | 2 +-
bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/786706ba/bigtop-packages/src/deb/hcatalog/hcatalog.preinst
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/hcatalog/hcatalog.preinst b/bigtop-packages/src/deb/hcatalog/hcatalog.preinst
index acaca99..31d76e6 100644
--- a/bigtop-packages/src/deb/hcatalog/hcatalog.preinst
+++ b/bigtop-packages/src/deb/hcatalog/hcatalog.preinst
@@ -26,7 +26,7 @@ case "$1" in
adduser \
--system \
--group \
- --home /usr/lib/hcatalog \
+ --home /var/run/hcatalog \
--gecos "HCatalog User" \
--shell /bin/false \
hcatalog >/dev/null
http://git-wip-us.apache.org/repos/asf/bigtop/blob/786706ba/bigtop-packages/src/rpm/hcatalog/SPECS/hcatalog.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/hcatalog/SPECS/hcatalog.spec b/bigtop-packages/src/rpm/hcatalog/SPECS/hcatalog.spec
index 3fd9889..f63e8fe 100644
--- a/bigtop-packages/src/rpm/hcatalog/SPECS/hcatalog.spec
+++ b/bigtop-packages/src/rpm/hcatalog/SPECS/hcatalog.spec
@@ -184,7 +184,7 @@ fi
%pre
getent group hcatalog >/dev/null || groupadd -r hcatalog || :
-getent passwd hcatalog >/dev/null || useradd -c "HCatalog User" -s /sbin/nologin -g hcatalog -r -d %{usr_lib_hcatalog} hcatalog 2> /dev/null || :
+getent passwd hcatalog >/dev/null || useradd -c "HCatalog User" -s /sbin/nologin -g hcatalog -r -d /var/run/hcatalog hcatalog 2> /dev/null || :
%post
%{alternatives_cmd} --install %{conf_hcatalog} hcatalog-conf %{conf_hcatalog}.dist 30
http://git-wip-us.apache.org/repos/asf/bigtop/blob/786706ba/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec b/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec
index 6cd98c2..0e880d3 100644
--- a/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec
+++ b/bigtop-packages/src/rpm/sqoop/SPECS/sqoop.spec
@@ -51,7 +51,7 @@ Name: sqoop
Version: %{sqoop_version}
Release: %{sqoop_release}
Summary: Tool for easy imports and exports of data sets between databases and the Hadoop ecosystem
-URL: http://incubator.apache.org/sqoop/
+URL: http://sqoop.apache.org
Group: Development/Libraries
Buildroot: %{_topdir}/INSTALL/%{name}-%{version}
License: APL2
[6/6] git commit: BIGTOP-958. update deployment code for HCatalog
Posted by rv...@apache.org.
BIGTOP-958. update deployment code for HCatalog
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/39af29da
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/39af29da
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/39af29da
Branch: refs/heads/master
Commit: 39af29da43fca8513a1ccca8aa01bc276c62d13a
Parents: a36d79e
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Wed May 1 11:30:38 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Wed May 1 14:52:08 2013 -0700
----------------------------------------------------------------------
bigtop-deploy/puppet/manifests/cluster.pp | 8 +-
.../puppet/modules/hcatalog/manifests/init.pp | 54 ++++
.../modules/hcatalog/templates/hcatalog-server | 8 +
.../puppet/modules/hcatalog/templates/webhcat.xml | 228 +++++++++++++++
.../puppet/modules/hcatalog/tests/init.pp | 17 +
5 files changed, 314 insertions(+), 1 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index 605c6a9..11aeb24 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -229,12 +229,18 @@ class hadoop_head_node inherits hadoop_worker_node {
kerberos_realm => $kerberos_realm,
}
-
hadoop-sqoop::server { "sqoop server":
}
hadoop-sqoop::client { "sqoop client":
}
+ hcatalog::server { "hcatalog server":
+ kerberos_realm => $kerberos_realm,
+ }
+ hcatalog::webhcat::server { "webhcat server":
+ kerberos_realm => $kerberos_realm,
+ }
+
hue::server { "hue server":
rm_url => $hadoop_rm_url,
rm_proxy_url => $hadoop_rm_proxy_url,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp b/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp
new file mode 100644
index 0000000..f9c07aa
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hcatalog {
+ define server($port = "9083", $kerberos_realm = "") {
+ package { "hcatalog-server":
+ ensure => latest,
+ }
+
+ file { "/etc/default/hcatalog-server":
+ content => template("hcatalog/hcatalog-server"),
+ require => Package["hcatalog-server"],
+ }
+
+ service { "hcatalog-server":
+ ensure => running,
+ require => [ Package["hcatalog-server"], File["/etc/default/hcatalog-server"] ],
+ hasrestart => true,
+ hasstatus => true,
+ }
+ }
+
+ class webhcat {
+ define server($port = "50111", $kerberos_realm = "") {
+ package { "webhcat-server":
+ ensure => latest,
+ }
+
+ file { "/etc/webhcat/conf/webhcat.xml":
+ content => template("hcatalog/webhcat.xml"),
+ require => Package["webhcat-server"],
+ }
+
+ service { "webhcat-server":
+ ensure => running,
+ require => [ Package["webhcat-server"], File["/etc/webhcat/conf/webhcat.xml"] ],
+ hasrestart => true,
+ hasstatus => true,
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server b/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server
new file mode 100644
index 0000000..7291c51
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hcatalog/templates/hcatalog-server
@@ -0,0 +1,8 @@
+export METASTORE_PORT=<%= @port %>
+
+export HADOOP_PREFIX=/usr
+export HIVE_HOME=/usr/lib/hive
+export HCAT_LOG_DIR=/var/log/hcatalog
+export HCAT_PID_DIR=/var/run/hcatalog
+#FIXME: workaround for HIVE-2757 (NOTE: HADOOP_HOME is obsolete for Hadoop .23)
+export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml b/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml
new file mode 100644
index 0000000..42c5aee
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hcatalog/templates/webhcat.xml
@@ -0,0 +1,228 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<!-- The default settings for Templeton. -->
+<!-- Edit templeton-site.xml to change settings for your local -->
+<!-- install. -->
+
+<configuration>
+
+ <property>
+ <name>templeton.port</name>
+ <value><%= @port %></value>
+ <description>The HTTP port for the main server.</description>
+ </property>
+
+ <property>
+ <name>templeton.hadoop.conf.dir</name>
+ <value>${env.HADOOP_CONF_DIR}</value>
+ <description>The path to the Hadoop configuration.</description>
+ </property>
+
+ <property>
+ <name>templeton.jar</name>
+ <value>${env.TEMPLETON_HOME}/share/webhcat/svr/webhcat-0.5.0-SNAPSHOT.jar</value>
+ <description>The path to the Templeton jar file.</description>
+ </property>
+
+ <property>
+ <name>templeton.libjars</name>
+ <value>${env.TEMPLETON_HOME}/share/webhcat/svr/lib/zookeeper-3.4.3.jar</value>
+ <description>Jars to add to the classpath.</description>
+ </property>
+
+ <property>
+ <name>templeton.override.jars</name>
+ <value>hdfs:///user/templeton/ugi.jar</value>
+ <description>
+ Jars to add the the HADOOP_CLASSPATH for all Map Reduce jobs.
+ This is a list of jars that must exist on hdfs that are added
+ to the Distributed Cache.
+ </description>
+ </property>
+
+ <property>
+ <name>templeton.override.enabled</name>
+ <value>false</value>
+ <description>
+ Enable the override path in templeton.override.jars
+ </description>
+ </property>
+
+ <property>
+ <name>templeton.streaming.jar</name>
+ <value>hdfs:///user/templeton/hadoop-streaming.jar</value>
+ <description>The hdfs path to the Hadoop streaming jar file.</description>
+ </property>
+
+ <property>
+ <name>templeton.hadoop</name>
+ <value>${env.HADOOP_PREFIX}/bin/hadoop</value>
+ <description>The path to the Hadoop executable.</description>
+ </property>
+
+ <property>
+ <name>templeton.pig.archive</name>
+ <value>hdfs:///apps/templeton/pig-0.10.1.tar.gz</value>
+ <description>The path to the Pig archive.</description>
+ </property>
+
+ <property>
+ <name>templeton.pig.path</name>
+ <value>pig-0.10.1.tar.gz/pig-0.10.1/bin/pig</value>
+ <description>The path to the Pig executable.</description>
+ </property>
+
+ <property>
+ <name>templeton.hcat</name>
+ <value>${env.HCAT_PREFIX}/bin/hcat</value>
+ <description>The path to the hcatalog executable.</description>
+ </property>
+
+ <property>
+ <name>templeton.hive.archive</name>
+ <value>hdfs:///apps/templeton/hive-0.10.0.tar.gz</value>
+ <description>The path to the Hive archive.</description>
+ </property>
+
+ <property>
+ <name>templeton.hive.path</name>
+ <value>hive-0.10.0.tar.gz/hive-0.10.0/bin/hive</value>
+ <description>The path to the Hive executable.</description>
+ </property>
+
+ <property>
+ <name>templeton.hive.properties</name>
+ <value>hive.metastore.local=false,hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false</value>
+ <description>Properties to set when running hive.</description>
+ </property>
+
+ <property>
+ <name>templeton.exec.encoding</name>
+ <value>UTF-8</value>
+ <description>The encoding of the stdout and stderr data.</description>
+ </property>
+
+ <property>
+ <name>templeton.exec.timeout</name>
+ <value>10000</value>
+ <description>
+ How long in milliseconds a program is allowed to run on the
+ Templeton box.
+ </description>
+ </property>
+
+ <property>
+ <name>templeton.exec.max-procs</name>
+ <value>16</value>
+ <description>The maximum number of processes allowed to run at once.</description>
+ </property>
+
+ <property>
+ <name>templeton.exec.max-output-bytes</name>
+ <value>1048576</value>
+ <description>
+ The maximum number of bytes from stdout or stderr stored in ram.
+ </description>
+ </property>
+
+ <property>
+ <name>templeton.controller.mr.child.opts</name>
+ <value>-server -Xmx256m -Djava.net.preferIPv4Stack=true</value>
+ <description>Java options to be passed to templeton controller map task.
+ The default value of mapreduce child "-Xmx" (heap memory limit)
+ might be close to what is allowed for a map task.
+ Even if templeton controller map task does not need much
+ memory, the jvm (with -server option?)
+ allocates the max memory when it starts. This along with the
+ memory used by pig/hive client it starts can end up exceeding
+ the max memory configured to be allowed for a map task
+ Use this option to set -Xmx to lower value
+ </description>
+ </property>
+
+ <property>
+ <name>templeton.exec.envs</name>
+ <value>HADOOP_PREFIX,HADOOP_HOME,JAVA_HOME,HIVE_HOME</value>
+ <description>The environment variables passed through to exec.</description>
+ </property>
+
+ <property>
+ <name>templeton.zookeeper.hosts</name>
+ <value>127.0.0.1:2181</value>
+ <description>ZooKeeper servers, as comma separated host:port pairs</description>
+ </property>
+
+ <property>
+ <name>templeton.zookeeper.session-timeout</name>
+ <value>30000</value>
+ <description>ZooKeeper session timeout in milliseconds</description>
+ </property>
+
+ <property>
+ <name>templeton.callback.retry.interval</name>
+ <value>10000</value>
+ <description>How long to wait between callback retry attempts in milliseconds</description>
+ </property>
+
+ <property>
+ <name>templeton.callback.retry.attempts</name>
+ <value>5</value>
+ <description>How many times to retry the callback</description>
+ </property>
+
+ <property>
+ <name>templeton.storage.class</name>
+ <value>org.apache.hcatalog.templeton.tool.HDFSStorage</value>
+ <description>The class to use as storage</description>
+ </property>
+
+ <property>
+ <name>templeton.storage.root</name>
+ <value>/templeton-hadoop</value>
+ <description>The path to the directory to use for storage</description>
+ </property>
+
+ <property>
+ <name>templeton.hdfs.cleanup.interval</name>
+ <value>43200000</value>
+ <description>The maximum delay between a thread's cleanup checks</description>
+ </property>
+
+ <property>
+ <name>templeton.hdfs.cleanup.maxage</name>
+ <value>604800000</value>
+ <description>The maximum age of a templeton job</description>
+ </property>
+
+ <property>
+ <name>templeton.zookeeper.cleanup.interval</name>
+ <value>43200000</value>
+ <description>The maximum delay between a thread's cleanup checks</description>
+ </property>
+
+ <property>
+ <name>templeton.zookeeper.cleanup.maxage</name>
+ <value>604800000</value>
+ <description>The maximum age of a templeton job</description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/39af29da/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp b/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp
new file mode 100644
index 0000000..59920ae
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hcatalog/tests/init.pp
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hcatalog::server { "test-hcatalog-server": }
+hcatalog::webhcat::server { "test-hcatalog-webhcat-server": }