You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by fr...@apache.org on 2012/03/15 22:52:00 UTC
svn commit: r1301223 - in /whirr/trunk: ./ recipes/ services/hama/
services/hama/src/main/java/org/apache/whirr/service/hama/
services/hama/src/main/resources/ services/hama/src/main/resources/functions/
services/hama/src/test/java/org/apache/whirr/ser...
Author: frankscholten
Date: Thu Mar 15 21:51:59 2012
New Revision: 1301223
URL: http://svn.apache.org/viewvc?rev=1301223&view=rev
Log:
WHIRR-335. Update Hama service to Hama 0.4-incubating
Modified:
whirr/trunk/CHANGES.txt
whirr/trunk/pom.xml
whirr/trunk/recipes/hama.properties
whirr/trunk/services/hama/pom.xml
whirr/trunk/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java
whirr/trunk/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java
whirr/trunk/services/hama/src/main/resources/functions/configure_hama.sh
whirr/trunk/services/hama/src/main/resources/functions/install_hama.sh
whirr/trunk/services/hama/src/main/resources/functions/start_hama.sh
whirr/trunk/services/hama/src/main/resources/whirr-hama-default.properties
whirr/trunk/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceController.java
whirr/trunk/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceTest.java
whirr/trunk/services/hama/src/test/resources/whirr-hama-test.properties
Modified: whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/whirr/trunk/CHANGES.txt?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/CHANGES.txt (original)
+++ whirr/trunk/CHANGES.txt Thu Mar 15 21:51:59 2012
@@ -9,6 +9,8 @@ Trunk (unreleased changes)
WHIRR-531. Add Pig as a service (Marco Didonna, Andrei Savu)
+ WHIRR-335. Update Hama service to Hama 0.4-incubating (Edward J. Yoon via Frank Scholten)
+
IMPROVEMENTS
WHIRR-514. Avoid hardcoded Whirr version number in karaf tests (kve)
Modified: whirr/trunk/pom.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/pom.xml?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/pom.xml (original)
+++ whirr/trunk/pom.xml Thu Mar 15 21:51:59 2012
@@ -68,7 +68,7 @@
<guava.version>11.0.1</guava.version>
<hadoop.version>0.20.205.0</hadoop.version>
<hamcrest.version>1.1</hamcrest.version>
- <hama.version>0.3.0-incubating</hama.version>
+ <hama.version>0.4.0-incubating</hama.version>
<hbase.version>0.89.20100924-28</hbase.version>
<jackson.version>1.5.2</jackson.version>
<jclouds.version>1.3.2</jclouds.version>
Modified: whirr/trunk/recipes/hama.properties
URL: http://svn.apache.org/viewvc/whirr/trunk/recipes/hama.properties?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/recipes/hama.properties (original)
+++ whirr/trunk/recipes/hama.properties Thu Mar 15 21:51:59 2012
@@ -69,4 +69,4 @@ whirr.instance-templates=1 zookeeper+had
# Expert: specify the version of Hama to install.
#
-#whirr.hama.tarball.url=http://archive.apache.org/dist/incubator/hama/0.3-incubating/hama-0.3.0-incubating.tar.gz
+#whirr.hama.tarball.url=http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz
Modified: whirr/trunk/services/hama/pom.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hama/pom.xml?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/services/hama/pom.xml (original)
+++ whirr/trunk/services/hama/pom.xml Thu Mar 15 21:51:59 2012
@@ -29,6 +29,8 @@
<version>0.8.0-SNAPSHOT</version>
<name>Apache Whirr Hama</name>
<properties>
+ <hadoop.version>0.20.2</hadoop.version>
+ <zookeeper.version>3.3.3</zookeeper.version>
<osgi.import>
!org.apache.whirr.service.hama*,
org.apache.commons.configuration*;version="[1.6,2)",
Modified: whirr/trunk/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java (original)
+++ whirr/trunk/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java Thu Mar 15 21:51:59 2012
@@ -51,7 +51,7 @@ public class HamaGroomServerClusterActio
InetAddress masterPublicAddress = instance.getPublicAddress();
event.getFirewallManager().addRules(
- Rule.create().destination(instance).ports(61000, GROOMSERVER_PORT));
+ Rule.create().destination(instance).ports(GROOMSERVER_PORT));
String hamaConfigureFunction = getConfiguration(clusterSpec).getString(
HamaConstants.KEY_CONFIGURE_FUNCTION,
Modified: whirr/trunk/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java (original)
+++ whirr/trunk/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java Thu Mar 15 21:51:59 2012
@@ -97,9 +97,9 @@ public class HamaMasterClusterActionHand
Instance instance = cluster.getInstanceMatching(role(ROLE));
InetAddress masterPublicAddress = instance.getPublicAddress();
- LOG.info("BSPMaster web UI available at http://{}:{}",
- masterPublicAddress.getHostName(), MASTER_WEB_UI_PORT);
-
+ LOG.info("BSPMaster web UI available at http://{}:{}", masterPublicAddress
+ .getHostName(), MASTER_WEB_UI_PORT);
+
String quorum = ZooKeeperCluster.getHosts(cluster);
Properties config = createClientSideProperties(masterPublicAddress, quorum);
createClientSideHadoopSiteFile(clusterSpec, config);
@@ -115,6 +115,9 @@ public class HamaMasterClusterActionHand
config.setProperty("bsp.master.address", master.getHostName() + ":"
+ MASTER_PORT);
+ config.setProperty("fs.default.name", "hdfs://" + master.getHostName()
+ + ":8020");
+
config.setProperty("hadoop.socks.server", "localhost:6666");
config.setProperty("hadoop.rpc.socket.factory.class.default",
"org.apache.hadoop.net.SocksSocketFactory");
Modified: whirr/trunk/services/hama/src/main/resources/functions/configure_hama.sh
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hama/src/main/resources/functions/configure_hama.sh?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/services/hama/src/main/resources/functions/configure_hama.sh (original)
+++ whirr/trunk/services/hama/src/main/resources/functions/configure_hama.sh Thu Mar 15 21:51:59 2012
@@ -41,7 +41,7 @@ function configure_hama() {
done
# assign default URL if no other given (optional)
- HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.3-incubating/hama-0.3.0-incubating.tar.gz}
+ HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz}
# derive details from the URL
HAMA_TAR_FILE=${HAMA_TAR_URL##*/}
Modified: whirr/trunk/services/hama/src/main/resources/functions/install_hama.sh
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hama/src/main/resources/functions/install_hama.sh?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/services/hama/src/main/resources/functions/install_hama.sh (original)
+++ whirr/trunk/services/hama/src/main/resources/functions/install_hama.sh Thu Mar 15 21:51:59 2012
@@ -36,7 +36,7 @@ function install_hama() {
done
# assign default URL if no other given (optional)
- HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.3-incubating/hama-0.3.0-incubating.tar.gz}
+ HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz}
# derive details from the URL
HAMA_TAR_FILE=${HAMA_TAR_URL##*/}
HAMA_TAR_MD5_FILE=$HAMA_TAR_FILE.md5
Modified: whirr/trunk/services/hama/src/main/resources/functions/start_hama.sh
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hama/src/main/resources/functions/start_hama.sh?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/services/hama/src/main/resources/functions/start_hama.sh (original)
+++ whirr/trunk/services/hama/src/main/resources/functions/start_hama.sh Thu Mar 15 21:51:59 2012
@@ -34,7 +34,7 @@ function start_hama() {
done
# assign default URL if no other given (optional)
- HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.3-incubating/hama-0.3.0-incubating.tar.gz}
+ HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz}
# derive details from the URL
HAMA_TAR_FILE=${HAMA_TAR_URL##*/}
Modified: whirr/trunk/services/hama/src/main/resources/whirr-hama-default.properties
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hama/src/main/resources/whirr-hama-default.properties?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/services/hama/src/main/resources/whirr-hama-default.properties (original)
+++ whirr/trunk/services/hama/src/main/resources/whirr-hama-default.properties Thu Mar 15 21:51:59 2012
@@ -10,4 +10,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-whirr.hama.tarball.url=http://archive.apache.org/dist/incubator/hama/0.3-incubating/hama-0.3.0-incubating.tar.gz
+whirr.hadoop.version=0.20.2
+whirr.hama.tarball.url=http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz
Modified: whirr/trunk/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceController.java
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceController.java?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceController.java (original)
+++ whirr/trunk/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceController.java Thu Mar 15 21:51:59 2012
@@ -86,18 +86,20 @@ public class HamaServiceController {
private static void waitForGroomServers(BSPJobClient client) throws IOException {
while (true) {
- ClusterStatus clusterStatus = client.getClusterStatus(true);
- int grooms = clusterStatus.getGroomServers();
- if (grooms > 0) {
- LOG.info("{} groomservers reported in. Continuing.", grooms);
- break;
- }
try {
System.out.print(".");
Thread.sleep(1000);
} catch (InterruptedException e) {
break;
}
+
+ ClusterStatus clusterStatus = client.getClusterStatus(true);
+ int grooms = clusterStatus.getActiveGroomServerStatus().size();
+
+ if (grooms > 0) {
+ LOG.info("{} groomservers reported in. Continuing.", grooms);
+ break;
+ }
}
}
Modified: whirr/trunk/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceTest.java
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceTest.java?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceTest.java (original)
+++ whirr/trunk/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceTest.java Thu Mar 15 21:51:59 2012
@@ -17,26 +17,39 @@
*/
package org.apache.whirr.service.hama.integration;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.Text;
import org.apache.hama.HamaConfiguration;
import org.apache.hama.bsp.BSPJob;
import org.apache.hama.bsp.BSPJobClient;
+import org.apache.hama.bsp.BSPJobID;
import org.apache.hama.bsp.ClusterStatus;
-import org.apache.hama.examples.PiEstimator;
+import org.apache.hama.bsp.FileOutputFormat;
+import org.apache.hama.bsp.NullInputFormat;
+import org.apache.hama.bsp.RunningJob;
+import org.apache.hama.bsp.TextOutputFormat;
import org.apache.hama.examples.PiEstimator.MyEstimator;
import org.apache.whirr.TestConstants;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HamaServiceTest {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(HamaServiceTest.class);
+
private static HamaServiceController controller = HamaServiceController
.getInstance();
- private static String masterTask = "master.task.";
-
+
+ private final static Path TMP_OUTPUT = new Path("/pi-"
+ + System.currentTimeMillis());
+
@BeforeClass
public static void setUp() throws Exception {
controller.ensureClusterRunning();
@@ -49,27 +62,40 @@ public class HamaServiceTest {
@Test(timeout = TestConstants.ITEST_TIMEOUT)
public void test() throws Exception {
- HamaConfiguration conf = controller.getConfiguration();
+ HamaConfiguration jobConf = controller.getConfiguration();
+ jobConf.set("hadoop.rpc.socket.factory.class.default",
+ "org.apache.hadoop.net.StandardSocketFactory");
+
+ BSPJob bsp = new BSPJob(jobConf, new BSPJobID());
+ LOG.info("Job conf: "
+ + bsp.getConf().get("hadoop.rpc.socket.factory.class.default") + ", "
+ + bsp.getJobID().toString());
+
+ bsp.setJarByClass(MyEstimator.class);
+ bsp.setBspClass(MyEstimator.class);
+ bsp.setInputFormat(NullInputFormat.class);
+ bsp.setOutputKeyClass(Text.class);
+ bsp.setOutputValueClass(DoubleWritable.class);
+ bsp.setOutputFormat(TextOutputFormat.class);
+ bsp.set("bsp.working.dir", "/tmp");
+ FileOutputFormat.setOutputPath(bsp, TMP_OUTPUT);
+
+ LOG.info("Client configuration start ..");
- BSPJobClient jobClient = new BSPJobClient(conf);
+ HamaConfiguration clientConf = controller.getConfiguration();
+ BSPJobClient jobClient = new BSPJobClient(clientConf);
ClusterStatus cluster = jobClient.getClusterStatus(true);
assertNotNull(cluster);
assertTrue(cluster.getGroomServers() > 0);
+ assertTrue(cluster.getMaxTasks() > 1);
+ bsp.setNumBspTask(cluster.getMaxTasks());
- BSPJob bsp = new BSPJob(conf, PiEstimator.class);
- // Set the job name
- bsp.setJobName("Pi Estimation Example");
- bsp.setBspClass(MyEstimator.class);
- bsp.setNumBspTask(cluster.getGroomServers());
+ LOG.info("Client conf: "
+ + clientConf.get("hadoop.rpc.socket.factory.class.default"));
- for (String peerName : cluster.getActiveGroomNames().values()) {
- conf.set(masterTask, peerName);
- break;
- }
-
- if (bsp.waitForCompletion(true)) {
- assertEquals(jobClient.getAllJobs().length, 1);
- }
+ RunningJob rJob = jobClient.submitJob(bsp);
+ rJob.waitForCompletion();
+ LOG.info("finished");
}
}
Modified: whirr/trunk/services/hama/src/test/resources/whirr-hama-test.properties
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hama/src/test/resources/whirr-hama-test.properties?rev=1301223&r1=1301222&r2=1301223&view=diff
==============================================================================
--- whirr/trunk/services/hama/src/test/resources/whirr-hama-test.properties (original)
+++ whirr/trunk/services/hama/src/test/resources/whirr-hama-test.properties Thu Mar 15 21:51:59 2012
@@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+whirr.hadoop.version=0.20.2
whirr.cluster-name=hama-itest-${sys:whirr.test.provider}-${sys:user.name}
whirr.instance-templates=1 zookeeper,1 hadoop-namenode+hadoop-jobtracker+hama-master,1 hadoop-datanode+hadoop-tasktracker+hama-groomserver
@@ -22,5 +23,5 @@ whirr.provider=${sys:whirr.test.provider
whirr.identity=${sys:whirr.test.identity}
whirr.credential=${sys:whirr.test.credential}
-whirr.hama.tarball.url=http://archive.apache.org/dist/incubator/hama/0.3-incubating/hama-0.3.0-incubating.tar.gz
+whirr.hama.tarball.url=http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz