You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2012/07/12 06:09:18 UTC
svn commit: r1360521 [4/7] - in /hama/site/trunk: ./ css/ files/ images/
images/profiles/ js/
Modified: hama/site/trunk/getting_started_with_hama.html
URL: http://svn.apache.org/viewvc/hama/site/trunk/getting_started_with_hama.html?rev=1360521&r1=1360520&r2=1360521&view=diff
==============================================================================
--- hama/site/trunk/getting_started_with_hama.html (original)
+++ hama/site/trunk/getting_started_with_hama.html Thu Jul 12 04:09:17 2012
@@ -1,278 +1,323 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<!-- Generated by Apache Maven Doxia at Jun 19, 2012 -->
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
- <head>
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!--
+ | Generated by Apache Maven Doxia at Jul 12, 2012
+ | Rendered using Apache Maven Fluido Skin 1.2.3-SNAPSHOT
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Hama -
- Getting Started with Hama</title>
- <style type="text/css" media="all">
- @import url("./css/maven-base.css");
- @import url("./css/maven-theme.css");
- @import url("./css/site.css");
- </style>
- <link rel="stylesheet" href="./css/print.css" type="text/css" media="print" />
- <meta name="Date-Revision-yyyymmdd" content="20120619" />
- <meta http-equiv="Content-Language" content="en" />
-
- </head>
- <body class="composite">
- <div id="banner">
- <a href="index.html" id="bannerLeft">
- <img src="images/headerlogo.png" alt="Hama" />
- </a>
- <div class="clear">
- <hr/>
- </div>
- </div>
- <div id="breadcrumbs">
-
-
- <div class="xleft">
- <a href="http://www.apache.org/" class="externalLink" title="Apache">Apache</a>
- >
- <a href="http://hama.apache.org/" class="externalLink" title="Hama">Hama</a>
- </div>
- <div class="xright"> <a href="https://wiki.apache.org/hama" class="externalLink" title="Wiki">Wiki</a>
- |
- <a href="http://blogs.apache.org/hama/" class="externalLink" title="Blog">Blog</a>
- |
- <a href="https://issues.apache.org/jira/browse/HAMA" class="externalLink" title="JIRA">JIRA</a>
- |
- <a href="https://svn.apache.org/repos/asf/hama/" class="externalLink" title="SVN">SVN</a>
- |
- <a href="http://twitter.com/apachehama" class="externalLink" title="Twitter">Twitter</a>
-
-
- | <span id="publishDate">Last Published: 2012-06-19</span>
- | <span id="projectVersion">Version: 0.5.0-SNAPSHOT</span>
- </div>
- <div class="clear">
- <hr/>
- </div>
- </div>
- <div id="leftColumn">
- <div id="navcolumn">
-
-
- <h5>Information</h5>
- <ul>
- <li class="none">
- <a href="index.html" title="Overview">Overview</a>
- </li>
- <li class="none">
- <a href="downloads.html" title="Downloads">Downloads</a>
- </li>
- <li class="none">
- <a href="team-list.html" title="Who We Are">Who We Are</a>
- </li>
- <li class="none">
- <a href="privacy_policy.html" title="Privacy Policy">Privacy Policy</a>
- </li>
- </ul>
- <h5>Resources</h5>
- <ul>
- <li class="none">
- <a href="developers.html" title="Developers">Developers</a>
- </li>
- <li class="none">
- <a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a>
- </li>
- <li class="none">
- <a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a>
- </li>
- <li class="none">
- <a href="irc-channel.html" title="IRC channel">IRC channel</a>
- </li>
- </ul>
- <h5>Documentation</h5>
- <ul>
- <li class="none">
- <strong>Getting Started</strong>
- </li>
- <li class="none">
- <a href="hama_on_clouds.html" title="Hama on Clouds">Hama on Clouds</a>
- </li>
- <li class="none">
- <a href="hama_bsp_tutorial.html" title="Hama BSP tutorial">Hama BSP tutorial</a>
- </li>
- <li class="none">
- <a href="hama_graph_tutorial.html" title="Hama Graph tutorial">Hama Graph tutorial</a>
- </li>
- <li class="none">
- <a href="http://wiki.apache.org/hama/Benchmarks" class="externalLink" title="Benchmarks">Benchmarks</a>
- </li>
- <li class="none">
- <a href="http://wiki.apache.org/hama/Presentations" class="externalLink" title="Presentations">Presentations</a>
- </li>
- <li class="none">
- <a href="http://wiki.apache.org/hama/Articles" class="externalLink" title="Research Papers">Research Papers</a>
- </li>
- <li class="none">
- <a href="http://wiki.apache.org/hama/Books" class="externalLink" title="Books">Books</a>
- </li>
- </ul>
- <h5>Related Projects</h5>
- <ul>
- <li class="none">
- <a href="http://hadoop.apache.org/" class="externalLink" title="Hadoop">Hadoop</a>
- </li>
- <li class="none">
- <a href="https://whirr.apache.org/" class="externalLink" title="Whirr">Whirr</a>
- </li>
- <li class="none">
- <a href="https://incubator.apache.org/bigtop/" class="externalLink" title="Bigtop">Bigtop</a>
- </li>
- <li class="none">
- <a href="http://mahout.apache.org/" class="externalLink" title="Mahout">Mahout</a>
- </li>
- </ul>
- <h5>Art works</h5>
- <ul>
- <li class="none">
- <a href="artwork.html" title="Artwork and Clothing">Artwork and Clothing</a>
- </li>
- </ul>
- <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
- <img class="poweredBy" alt="Built by Maven" src="./images/logos/maven-feather.png" />
- </a>
-
-
- </div>
- </div>
- <div id="bodyColumn">
- <div id="contentBox">
- <!-- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. -->
-
- <div class="section"><h2>Getting Started with Hama<a name="Getting_Started_with_Hama"></a></h2></div>
- <p>This document describes how to install, configure and manage Hama clusters ranging from a few nodes to extremely large clusters with thousands of nodes.</p>
-
- <div class="section"><h3>Requirements<a name="Requirements"></a></h3></div>
- <p>1. Make sure all required software is installed on all nodes in your cluster:</p>
- <ul>
- <li>hadoop-0.20.x (non-secure version)</li>
- <li>Sun Java JDK 1.6.x or higher version</li>
- <li>SSH access to manage BSP deamons</li>
- </ul>
- <p>2. <a href="downloads.html">Download</a> Hama from the release page.
- <br /><br />
- For additional information consult our
- <a class="externalLink" href="http://wiki.apache.org/hama/CompatibilityTable">Compatibility Table</a></p>
-
- <div class="section"><h3>Mode<a name="Mode"></a></h3></div>
- <p>Just like Hadoop, we distinct between three modes:</p>
- <ul>
- <li>Local Mode - <font style="font-size: 11px;">This mode is the default mode if you download Hama (>= 0.3.0) and install it. When submitting a job it will run a local multithreaded BSP Engine on your server. It can be configured via the bsp.master.address property to local. You can adjust the number of threads used in this utility by setting the <b>bsp.local.tasks.maximum property</b>. See the Settings step how and where to configure this.</font></li>
- <li>Pseudo Distributed Mode - <font style="font-size: 11px;">This mode is when you just have a single server and want to launch all the deamon processes (BSPMaster, Groom and Zookeeper). It can be configured when you set the bsp.master.address to a host address e.g., <b>localhost</b> and put the same address into the groomservers file in the configuration directory. As stated it will run a BSPMaster, a Groom and a Zookeeper on your machine.</font></li>
- <li>Distributed Mode - <font style="font-size: 11px;">This mode is just like the "Pseudo Distributed Mode", but you have multiple machines, which are mapped in the groomservers file.</font></li>
- </ul>
-
- <div class="section"><h3>Configuration Files<a name="Configuration_Files"></a></h3></div>
- <p>The $HAMA_HOME/conf directory contains some configuration files for Hama. These are:</p>
- <ul>
- <li>hama-env.sh - This file contains some environment variable settings used by Hama. You can use these to affect some aspects of Hama daemon behavior, such as where log files are stored, the maximum amount of heap used etc. The only variable you should need to change in this file is JAVA_HOME, which specifies the path to the Java 1.5.x installation used by Hama.</li>
- <li>groomservers - This file lists the hosts, one per line, where the GroomServer daemons will run. By default this contains the single entry localhost</li>
- <li>hama-default.xml - This file contains generic default settings for Hama daemons. Do not modify this file.</li>
- <li>hama-site.xml - This file contains site specific settings for all Hama daemons and BSP jobs. This file is empty by default. Settings in this file override those in hama-default.xml. This file should contain settings that must be respected by all servers and clients in a Hama installation.</li>
- </ul>
-
- <div class="section"><h3>Startup Scripts<a name="Startup_Scripts"></a></h3></div>
- <p>The $HAMA_HOME/bin directory contains some script used to start up the Hama daemons.</p>
- <ul><li>start-bspd.sh - Starts all Hama daemons, the BSPMaster, GroomServers and Zookeeper.</li></ul>
- <p>Note: You have to start Hama with the same user which is configured for Hadoop.</p>
-
- <div class="section"><h3>Setting up Hama<a name="Setting_up_Hama"></a></h3></div>
- <ul><li>BSPMaster and Zookeeper settings - Figure out where to run your HDFS namenode and BSPMaster. Set the variable bsp.master.address to the BSPMaster's intended host:port. Set the variable fs.default.name to the HDFS Namenode's intended host:port.</li></ul>
- <p>Here's an example of a hama-site.xml file:</p>
-
-<div class="green"><pre>
- <?xml version="1.0"?>
- <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
- <configuration>
- <property>
- <name>bsp.master.address</name>
- <value>host1.mydomain.com:40000</value>
- <description>The address of the bsp master server. Either the
- literal string "local" or a host:port for distributed mode
- </description>
- </property>
-
- <property>
- <name>fs.default.name</name>
- <value>hdfs://host1.mydomain.com:9000/</value>
- <description>
- The name of the default file system. Either the literal string
- "local" or a host:port for HDFS.
- </description>
- </property>
-
- <property>
- <name>hama.zookeeper.quorum</name>
- <value>host1.mydomain.com,host2.mydomain.com</value>
- <description>Comma separated list of servers in the ZooKeeper Quorum.
- For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
- By default this is set to localhost for local and pseudo-distributed modes
- of operation. For a fully-distributed setup, this should be set to a full
- list of ZooKeeper quorum servers. If HAMA_MANAGES_ZK is set in hama-env.sh
- this is the list of servers which we will start/stop zookeeper on.
- </description>
- </property>
- </configuration>
-</pre></div>
-
-<p>If you are managing your own ZooKeeper, you have to specify the port number as below:</p>
-
-<div class="green"><pre>
- <property>
- <name>hama.zookeeper.property.clientPort</name>
- <value>2181</value>
- </property>
-</pre></div>
-<p>See all <a class="externalLink" href="http://wiki.apache.org/hama/GettingStarted/Properties">Configuration Properties</a></p>
-
- <div class="section"><h3>Starting a Hama cluster<a name="Starting_a_Hama_cluster"></a></h3></div>
- <p>NOTE: Skip this step if you're in Local Mode.
- <br />Run the command:</p>
+ Getting Started with Hama</title>
+ <link rel="stylesheet" href="./css/apache-maven-fluido.min.css" />
+ <link rel="stylesheet" href="./css/site.css" />
+ <link rel="stylesheet" href="./css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="./js/apache-maven-fluido.min.js"></script>
+
- <div class="green"><pre>
- % $HAMA_HOME/bin/start-bspd.sh</pre></div>
- <p>This will startup a BSPMaster, GroomServers and Zookeeper on your machine.</p>
- <p>Run the command:</p>
- <div class="green"><pre>
- % $HAMA_HOME/bin/stop-bspd.sh</pre></div>
-
- <p>to stop all the daemons running on your cluster.</p>
- <div class="section"><h3>Execute Hama Examples<a name="Execute_Hama_Examples"></a></h3></div>
- <div class="green"><pre>
- % $HAMA_HOME/bin/hama jar hama-examples-x.x.x.jar [args]</pre></div>
+ <meta name="Date-Revision-yyyymmdd" content="20120712" />
+ <meta http-equiv="Content-Language" content="en" />
+ </head>
+ <body class="topBarDisabled">
+
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="index.html" id="bannerLeft">
+ <img src="images/headerlogo.png" alt="Hama"/>
+ </a>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org/" class="externalLink" title="Apache">Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://hama.apache.org/" class="externalLink" title="Hama">Hama</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Hama -
+ Getting Started with Hama</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published: 2012-07-12</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.6.0-SNAPSHOT</li>
+
+ <li class="divider pull-right">|</li>
+
+ <li class="pull-right"> <a href="http://twitter.com/apachehama" class="externalLink" title="Twitter">Twitter</a>
+ </li>
+
+ <li class="divider pull-right">|</li>
+
+ <li class="pull-right"> <a href="http://svn.apache.org/repos/asf/hama/" class="externalLink" title="SVN">SVN</a>
+ </li>
+
+ <li class="divider pull-right">|</li>
+
+ <li class="pull-right"> <a href="http://issues.apache.org/jira/browse/HAMA" class="externalLink" title="JIRA">JIRA</a>
+ </li>
+
+ <li class="divider pull-right">|</li>
+
+ <li class="pull-right"> <a href="http://blogs.apache.org/hama/" class="externalLink" title="Blog">Blog</a>
+ </li>
+
+ <li class="divider pull-right">|</li>
+
+ <li class="pull-right"> <a href="http://wiki.apache.org/hama" class="externalLink" title="Wiki">Wiki</a>
+ </li>
+
+ </ul>
+ </div>
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+
+
+ <h3>Information</h3>
+ <ul>
+ <li class="none">
+ <a href="index.html" title="Overview">Overview</a>
+ </li>
+ <li class="none">
+ <a href="downloads.html" title="Downloads">Downloads</a>
+ </li>
+ <li class="none">
+ <a href="team-list.html" title="Who We Are">Who We Are</a>
+ </li>
+ <li class="none">
+ <a href="privacy_policy.html" title="Privacy Policy">Privacy Policy</a>
+ </li>
+ </ul>
+ <h3>Resources</h3>
+ <ul>
+ <li class="none">
+ <a href="developers.html" title="Developers">Developers</a>
+ </li>
+ <li class="none">
+ <a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a>
+ </li>
+ <li class="none">
+ <a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a>
+ </li>
+ <li class="none">
+ <a href="irc-channel.html" title="IRC channel">IRC channel</a>
+ </li>
+ </ul>
+ <h3>Documentation</h3>
+ <ul>
+ <li class="none">
+ <strong>Getting Started</strong>
+ </li>
+ <li class="none">
+ <a href="hama_on_clouds.html" title="Hama on Clouds">Hama on Clouds</a>
+ </li>
+ <li class="none">
+ <a href="hama_bsp_tutorial.html" title="BSP tutorial">BSP tutorial</a>
+ </li>
+ <li class="none">
+ <a href="hama_graph_tutorial.html" title="Graph tutorial">Graph tutorial</a>
+ </li>
+ <li class="none">
+ <a href="http://wiki.apache.org/hama/Benchmarks" class="externalLink" title="Benchmarks">Benchmarks</a>
+ </li>
+ <li class="none">
+ <a href="http://wiki.apache.org/hama/Presentations" class="externalLink" title="Presentations">Presentations</a>
+ </li>
+ <li class="none">
+ <a href="http://wiki.apache.org/hama/Articles" class="externalLink" title="Research Papers">Research Papers</a>
+ </li>
+ <li class="none">
+ <a href="http://wiki.apache.org/hama/Books" class="externalLink" title="Books">Books</a>
+ </li>
+ </ul>
+ <h3>Related Projects</h3>
+ <ul>
+ <li class="none">
+ <a href="http://hadoop.apache.org/" class="externalLink" title="Hadoop">Hadoop</a>
+ </li>
+ <li class="none">
+ <a href="http://whirr.apache.org/" class="externalLink" title="Whirr">Whirr</a>
+ </li>
+ <li class="none">
+ <a href="http://incubator.apache.org/bigtop/" class="externalLink" title="Bigtop">Bigtop</a>
+ </li>
+ <li class="none">
+ <a href="http://mahout.apache.org/" class="externalLink" title="Mahout">Mahout</a>
+ </li>
+ </ul>
+ <h3>Art works</h3>
+ <ul>
+ <li class="none">
+ <a href="artwork.html" title="Artwork and Clothing">Artwork and Clothing</a>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="poweredBy" alt="Built by Maven" src="./images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+ <div id="bodyColumn" class="span10" >
+
+ <!-- Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. -->
+
+ <div class="section"><h2>Getting Started with Hama<a name="Getting_Started_with_Hama"></a></h2></div>
+ <p>This document describes how to install, configure and manage Hama clusters ranging from a few nodes to extremely large clusters with thousands of nodes.</p>
+
+ <div class="section"><h3>Requirements<a name="Requirements"></a></h3></div>
+ <p>1. Make sure all required software is installed on all nodes in your cluster:</p>
+ <ul>
+ <li>hadoop-0.20.x (non-secure version)</li>
+ <li>Sun Java JDK 1.6.x or higher version</li>
+ <li>SSH access to manage BSP deamons</li>
+ </ul>
+ <p>2. <a href="downloads.html">Download</a> Hama from the release page.
+ <br /><br />
+ For additional information consult our
+ <a class="externalLink" href="http://wiki.apache.org/hama/CompatibilityTable">Compatibility Table</a></p>
+
+ <div class="section"><h3>Mode<a name="Mode"></a></h3></div>
+ <p>Just like Hadoop, we distinct between three modes:</p>
+ <ul>
+ <li>Local Mode - <font style="font-size: 11px;">This mode is the default mode if you download Hama (>= 0.3.0) and install it. When submitting a job it will run a local multithreaded BSP Engine on your server. It can be configured via the bsp.master.address property to local. You can adjust the number of threads used in this utility by setting the <b>bsp.local.tasks.maximum property</b>. See the Settings step how and where to configure this.</font></li>
+ <li>Pseudo Distributed Mode - <font style="font-size: 11px;">This mode is when you just have a single server and want to launch all the deamon processes (BSPMaster, Groom and Zookeeper). It can be configured when you set the bsp.master.address to a host address e.g., <b>localhost</b> and put the same address into the groomservers file in the configuration directory. As stated it will run a BSPMaster, a Groom and a Zookeeper on your machine.</font></li>
+ <li>Distributed Mode - <font style="font-size: 11px;">This mode is just like the "Pseudo Distributed Mode", but you have multiple machines, which are mapped in the groomservers file.</font></li>
+ </ul>
+
+ <div class="section"><h3>Configuration Files<a name="Configuration_Files"></a></h3></div>
+ <p>The $HAMA_HOME/conf directory contains some configuration files for Hama. These are:</p>
+ <ul>
+ <li>hama-env.sh - This file contains some environment variable settings used by Hama. You can use these to affect some aspects of Hama daemon behavior, such as where log files are stored, the maximum amount of heap used etc. The only variable you should need to change in this file is JAVA_HOME, which specifies the path to the Java 1.5.x installation used by Hama.</li>
+ <li>groomservers - This file lists the hosts, one per line, where the GroomServer daemons will run. By default this contains the single entry localhost</li>
+ <li>hama-default.xml - This file contains generic default settings for Hama daemons. Do not modify this file.</li>
+ <li>hama-site.xml - This file contains site specific settings for all Hama daemons and BSP jobs. This file is empty by default. Settings in this file override those in hama-default.xml. This file should contain settings that must be respected by all servers and clients in a Hama installation.</li>
+ </ul>
+
+ <div class="section"><h3>Startup Scripts<a name="Startup_Scripts"></a></h3></div>
+ <p>The $HAMA_HOME/bin directory contains some script used to start up the Hama daemons.</p>
+ <ul><li>start-bspd.sh - Starts all Hama daemons, the BSPMaster, GroomServers and Zookeeper.</li></ul>
+ <p>Note: You have to start Hama with the same user which is configured for Hadoop.</p>
+
+ <div class="section"><h3>Setting up Hama<a name="Setting_up_Hama"></a></h3></div>
+ <ul><li>BSPMaster and Zookeeper settings - Figure out where to run your HDFS namenode and BSPMaster. Set the variable bsp.master.address to the BSPMaster's intended host:port. Set the variable fs.default.name to the HDFS Namenode's intended host:port.</li></ul>
+ <p>Here's an example of a hama-site.xml file:</p>
+
+<div><pre>
+ <?xml version="1.0"?>
+ <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+ <configuration>
+ <property>
+ <name>bsp.master.address</name>
+ <value>host1.mydomain.com:40000</value>
+ <description>The address of the bsp master server. Either the
+ literal string "local" or a host:port for distributed mode
+ </description>
+ </property>
+
+ <property>
+ <name>fs.default.name</name>
+ <value>hdfs://host1.mydomain.com:9000/</value>
+ <description>
+ The name of the default file system. Either the literal string
+ "local" or a host:port for HDFS.
+ </description>
+ </property>
- </div>
- </div>
- <div class="clear">
- <hr/>
- </div>
- <div id="footer">
- <div class="xright">Copyright © 2008-2012
- <a href="http://www.apache.org/">The Apache Software Foundation</a>.
- All Rights Reserved.
-
- </div>
- <div class="clear">
- <hr/>
- </div>
- </div>
- </body>
-</html>
+ <property>
+ <name>hama.zookeeper.quorum</name>
+ <value>host1.mydomain.com,host2.mydomain.com</value>
+ <description>Comma separated list of servers in the ZooKeeper Quorum.
+ For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
+ By default this is set to localhost for local and pseudo-distributed modes
+ of operation. For a fully-distributed setup, this should be set to a full
+ list of ZooKeeper quorum servers. If HAMA_MANAGES_ZK is set in hama-env.sh
+ this is the list of servers which we will start/stop zookeeper on.
+ </description>
+ </property>
+ </configuration>
+</pre></div>
+
+<p>If you are managing your own ZooKeeper, you have to specify the port number as below:</p>
+
+<div><pre>
+ <property>
+ <name>hama.zookeeper.property.clientPort</name>
+ <value>2181</value>
+ </property>
+</pre></div>
+<p>See all <a class="externalLink" href="http://wiki.apache.org/hama/GettingStarted/Properties">Configuration Properties</a></p>
+
+ <div class="section"><h3>Starting a Hama cluster<a name="Starting_a_Hama_cluster"></a></h3></div>
+ <p>NOTE: Skip this step if you're in Local Mode.
+ <br />Run the command:</p>
+
+ <div><pre>
+ % $HAMA_HOME/bin/start-bspd.sh</pre></div>
+ <p>This will startup a BSPMaster, GroomServers and Zookeeper on your machine.</p>
+ <p>Run the command:</p>
+ <div><pre>
+ % $HAMA_HOME/bin/stop-bspd.sh</pre></div>
+
+ <p>to stop all the daemons running on your cluster.</p>
+ <div class="section"><h3>Execute Hama Examples<a name="Execute_Hama_Examples"></a></h3></div>
+ <div><pre>
+ % $HAMA_HOME/bin/hama jar hama-examples-x.x.x.jar [args]</pre></div>
+
+
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2008-2012
+ <a href="http://www.apache.org/">The Apache Software Foundation</a>.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>
Modified: hama/site/trunk/hama_bsp_tutorial.html
URL: http://svn.apache.org/viewvc/hama/site/trunk/hama_bsp_tutorial.html?rev=1360521&r1=1360520&r2=1360521&view=diff
==============================================================================
--- hama/site/trunk/hama_bsp_tutorial.html (original)
+++ hama/site/trunk/hama_bsp_tutorial.html Thu Jul 12 04:09:17 2012
@@ -1,448 +1,493 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<!-- Generated by Apache Maven Doxia at Jun 19, 2012 -->
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
- <head>
- <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!--
+ | Generated by Apache Maven Doxia at Jul 12, 2012
+ | Rendered using Apache Maven Fluido Skin 1.2.3-SNAPSHOT
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Hama -
- BSP Tutorial</title>
- <style type="text/css" media="all">
- @import url("./css/maven-base.css");
- @import url("./css/maven-theme.css");
- @import url("./css/site.css");
- </style>
- <link rel="stylesheet" href="./css/print.css" type="text/css" media="print" />
- <meta name="Date-Revision-yyyymmdd" content="20120619" />
- <meta http-equiv="Content-Language" content="en" />
-
- </head>
- <body class="composite">
- <div id="banner">
- <a href="index.html" id="bannerLeft">
- <img src="images/headerlogo.png" alt="Hama" />
- </a>
- <div class="clear">
- <hr/>
- </div>
- </div>
- <div id="breadcrumbs">
-
-
- <div class="xleft">
- <a href="http://www.apache.org/" class="externalLink" title="Apache">Apache</a>
- >
- <a href="http://hama.apache.org/" class="externalLink" title="Hama">Hama</a>
- </div>
- <div class="xright"> <a href="https://wiki.apache.org/hama" class="externalLink" title="Wiki">Wiki</a>
- |
- <a href="http://blogs.apache.org/hama/" class="externalLink" title="Blog">Blog</a>
- |
- <a href="https://issues.apache.org/jira/browse/HAMA" class="externalLink" title="JIRA">JIRA</a>
- |
- <a href="https://svn.apache.org/repos/asf/hama/" class="externalLink" title="SVN">SVN</a>
- |
- <a href="http://twitter.com/apachehama" class="externalLink" title="Twitter">Twitter</a>
-
-
- | <span id="publishDate">Last Published: 2012-06-19</span>
- | <span id="projectVersion">Version: 0.5.0-SNAPSHOT</span>
- </div>
- <div class="clear">
- <hr/>
- </div>
- </div>
- <div id="leftColumn">
- <div id="navcolumn">
-
-
- <h5>Information</h5>
- <ul>
- <li class="none">
- <a href="index.html" title="Overview">Overview</a>
- </li>
- <li class="none">
- <a href="downloads.html" title="Downloads">Downloads</a>
- </li>
- <li class="none">
- <a href="team-list.html" title="Who We Are">Who We Are</a>
- </li>
- <li class="none">
- <a href="privacy_policy.html" title="Privacy Policy">Privacy Policy</a>
- </li>
- </ul>
- <h5>Resources</h5>
- <ul>
- <li class="none">
- <a href="developers.html" title="Developers">Developers</a>
- </li>
- <li class="none">
- <a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a>
- </li>
- <li class="none">
- <a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a>
- </li>
- <li class="none">
- <a href="irc-channel.html" title="IRC channel">IRC channel</a>
- </li>
- </ul>
- <h5>Documentation</h5>
- <ul>
- <li class="none">
- <a href="getting_started_with_hama.html" title="Getting Started">Getting Started</a>
- </li>
- <li class="none">
- <a href="hama_on_clouds.html" title="Hama on Clouds">Hama on Clouds</a>
- </li>
- <li class="none">
- <strong>Hama BSP tutorial</strong>
- </li>
- <li class="none">
- <a href="hama_graph_tutorial.html" title="Hama Graph tutorial">Hama Graph tutorial</a>
- </li>
- <li class="none">
- <a href="http://wiki.apache.org/hama/Benchmarks" class="externalLink" title="Benchmarks">Benchmarks</a>
- </li>
- <li class="none">
- <a href="http://wiki.apache.org/hama/Presentations" class="externalLink" title="Presentations">Presentations</a>
- </li>
- <li class="none">
- <a href="http://wiki.apache.org/hama/Articles" class="externalLink" title="Research Papers">Research Papers</a>
- </li>
- <li class="none">
- <a href="http://wiki.apache.org/hama/Books" class="externalLink" title="Books">Books</a>
- </li>
- </ul>
- <h5>Related Projects</h5>
- <ul>
- <li class="none">
- <a href="http://hadoop.apache.org/" class="externalLink" title="Hadoop">Hadoop</a>
- </li>
- <li class="none">
- <a href="https://whirr.apache.org/" class="externalLink" title="Whirr">Whirr</a>
- </li>
- <li class="none">
- <a href="https://incubator.apache.org/bigtop/" class="externalLink" title="Bigtop">Bigtop</a>
- </li>
- <li class="none">
- <a href="http://mahout.apache.org/" class="externalLink" title="Mahout">Mahout</a>
- </li>
- </ul>
- <h5>Art works</h5>
- <ul>
- <li class="none">
- <a href="artwork.html" title="Artwork and Clothing">Artwork and Clothing</a>
- </li>
- </ul>
- <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
- <img class="poweredBy" alt="Built by Maven" src="./images/logos/maven-feather.png" />
- </a>
-
-
- </div>
- </div>
- <div id="bodyColumn">
- <div id="contentBox">
- <!-- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. -->
-
- <div class="section"><h2>Hama BSP Tutorial<a name="Hama_BSP_Tutorial"></a></h2></div>
- <p>This document describes the Hama BSP framework and serves as a tutorial.</p>
- <div class="section"><h3>Overview<a name="Overview"></a></h3></div>
- <p>Hama provides a Pure BSP Bulk Synchronous Parallel Model for message passing and collective communication.
- A BSP program consists of a sequence of supersteps. Each superstep consists of the following three phases:</p>
-
- <ul>
- <li>Local computation</li>
- <li>Process communication</li>
- <li>Barrier synchronization</li>
- </ul>
- <p>BSP programming enables you to write high-performance parallel computing algorithms for a wide range of scientific problems.</p>
-
- <div class="section"><h3>Create your own BSP by extending BSP class<a name="Create_your_own_BSP_by_extending_BSP_class"></a></h3></div>
-
- <p>The way to create your own BSP class is to create a class that extends the org.apache.hama.bsp.<b>BSP</b> class.
- <br />
- The extending class must override the bsp() method, which is declared like this:
- </p>
- <div class="green"><pre>
- public abstract void bsp(BSPPeer<K1, V1, K2, V2, M extends Writable> peer) throws IOException,
- SyncException, InterruptedException;</pre></div>
-
- <p>
- You will define the BSP program inside this bsp() method.
- It is important to understand that it doesn't mean a single superstep.
- As described above, a BSP program consists of a sequence of supersteps.
- So it just gets called once, not all over again unlike Mapper or Reducer method.
- <br /><br />
- <b>NOTE</b>: Optionally, there are also setup() and cleanup() which will be called at the beginning of your computation,
- respectively at the end of the computation. cleanup() is guranteed to run after the computation or in case of failure.
- You can simply override the methods you need from BSP class.
- <br /><br />
- After your own BSP is created, you will need to configure a <b>BSPJob</b> and submit it to Hama cluster to execute a job.
- The BSP job configuration and submission interfaces is almost the same as the MapReduce job configuration:</p>
- <div class="green"><pre>
- HamaConfiguration conf = new HamaConfiguration();
- BSPJob job = new BSPJob(conf, MyBSP.class);
- job.setJobName("My BSP program");
- job.setBspClass(MyBSP.class);
- job.setInputFormat(NullInputFormat.class);
- job.setOutputKeyClass(Text.class);
- ...
- job.waitForCompletion(true);</pre></div>
- <p>See the below section for more detailed description of BSP user interfaces.</p>
+ BSP Tutorial</title>
+ <link rel="stylesheet" href="./css/apache-maven-fluido.min.css" />
+ <link rel="stylesheet" href="./css/site.css" />
+ <link rel="stylesheet" href="./css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="./js/apache-maven-fluido.min.js"></script>
+
- <div class="section"><h3>User Interfaces<a name="User_Interfaces"></a></h3></div>
- <div class="section"><div class="section"><div class="section"><h5>Inputs and Outputs<a name="Inputs_and_Outputs"></a></h5>
- <p>When setting up a BSPJob, you can provide a Input/OutputFormat and Paths like this:</p>
- <div class="green"><pre>
- job.setInputPath(new Path("/tmp/sequence.dat");
- job.setInputFormat(org.apache.hama.bsp.SequenceFileInputFormat.class);
- or,
- SequenceFileInputFormat.addInputPath(job, new Path("/tmp/sequence.dat"));
- or,
- SequenceFileInputFormat.addInputPaths(job, "/tmp/seq1.dat,/tmp/seq2.dat,/tmp/seq3.dat");
-
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
- job.setOutputFormat(TextOutputFormat.class);
- FileOutputFormat.setOutputPath(job, new Path("/tmp/result"));
- </pre></div>
+ <meta name="Date-Revision-yyyymmdd" content="20120712" />
+ <meta http-equiv="Content-Language" content="en" />
+ </head>
+ <body class="topBarDisabled">
+
+
- <p>Then, you can read the input and write the output from the methods in BSP class which has "BSPPeer" which contains an communication, counters, and IO interfaces as parameter.
- In this case we read a normal text file:</p>
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="index.html" id="bannerLeft">
+ <img src="images/headerlogo.png" alt="Hama"/>
+ </a>
+ </div>
+ <div class="pull-right"> </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="http://www.apache.org/" class="externalLink" title="Apache">Apache</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">
+ <a href="http://hama.apache.org/" class="externalLink" title="Hama">Hama</a>
+ </li>
+ <li class="divider ">/</li>
+ <li class="">Hama -
+ BSP Tutorial</li>
+
+
+
+ <li id="publishDate" class="pull-right">Last Published: 2012-07-12</li> <li class="divider pull-right">|</li>
+ <li id="projectVersion" class="pull-right">Version: 0.6.0-SNAPSHOT</li>
+
+ <li class="divider pull-right">|</li>
+
+ <li class="pull-right"> <a href="http://twitter.com/apachehama" class="externalLink" title="Twitter">Twitter</a>
+ </li>
- <div class="green"><pre>
- @Override
- public final void bsp(
- BSPPeer<LongWritable, Text, Text, LongWritable, Text> peer)
- throws IOException, InterruptedException, SyncException {
+ <li class="divider pull-right">|</li>
- // this method reads the next key value record from file
- KeyValuePair<LongWritable, Text> pair = peer.readNext();
+ <li class="pull-right"> <a href="http://svn.apache.org/repos/asf/hama/" class="externalLink" title="SVN">SVN</a>
+ </li>
- // the following lines do the same:
- LongWritable key = new LongWritable();
- Text value = new Text();
- peer.readNext(key, value);
+ <li class="divider pull-right">|</li>
- // write
- peer.write(value, key);
- }</pre></div>
- <p>Consult the docs for more detail on events like end of file.
- There is also a function which allows you to re-read the input from the beginning.
- This snippet reads the input five times:
- </p>
- <div class="green"><pre>
- for(int i = 0; i < 5; i++){
- LongWritable key = new LongWritable();
- Text value = new Text();
- while (peer.readNext(key, value)) {
- // read everything
- }
- // reopens the input
- peer.reopenInput()
- }</pre></div>
-
- </div><div class="section"><h5>Communication<a name="Communication"></a></h5>
- <p>Hama BSP provides simple but powerful communication APIs for many purposes.
- We tried to follow the standard library of BSP world as much as possible.
- The following table describes all the methods you can use:</p>
-
-<table class="bodyTable" align="center" border="0">
-<tr class="a"><td><b>Method</b></td><td><b>Description</b></td></tr>
-<tr class="b"><td>send(String peerName, BSPMessage msg)</td><td>Sends a message to another peer.</td></tr>
-<tr class="a"><td>getCurrentMessage()</td><td>Returns a received message.</td></tr>
-<tr class="b"><td>getNumCurrentMessages()</td><td>Returns the number of received messages.</td></tr>
-<tr class="a"><td>sync()</td><td>Barrier synchronization.</td></tr>
-<tr class="b"><td>getPeerName()</td><td>Returns a peer's hostname.</td></tr>
-<tr class="a"><td>getAllPeerNames()</td><td>Returns all peer's hostname.</td></tr>
-<tr class="b"><td>getSuperstepCount()</td><td>Returns the count of supersteps</td></tr>
-</table>
+ <li class="pull-right"> <a href="http://issues.apache.org/jira/browse/HAMA" class="externalLink" title="JIRA">JIRA</a>
+ </li>
- <p>The send() and all the other functions are very flexible. Here is an example that sends a message to all peers:</p>
-
- <div class="green"><pre>
- @Override
- public void bsp(
- BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, Text> peer)
- throws IOException, SyncException, InterruptedException {
- for (String peerName : peer.getAllPeerNames()) {
- peer.send(peerName,
- new Text("Hello from " + peer.getPeerName(), System.currentTimeMillis()));
- }
+ <li class="divider pull-right">|</li>
+
+ <li class="pull-right"> <a href="http://blogs.apache.org/hama/" class="externalLink" title="Blog">Blog</a>
+ </li>
- peer.sync();
- }</pre></div>
-
- </div><div class="section"><h5>Synchronization<a name="Synchronization"></a></h5>
+ <li class="divider pull-right">|</li>
+
+ <li class="pull-right"> <a href="http://wiki.apache.org/hama" class="externalLink" title="Wiki">Wiki</a>
+ </li>
- <p>When all the processes have entered the barrier via the sync() method,
- the Hama proceeds to the next superstep.
- In the previous example, the BSP job will be finished by one synchronization
- after sending a message "Hello from ..." to all peers.
- <br /><br />
- But, keep in mind that the sync() function is not the end of the BSP job.
- As was previously mentioned, all the communication functions are very flexible.
- For example, the sync() method also can be called in a for loop
- so that you can use to program the iterative methods sequentially:</p>
-
- <div class="green"><pre>
- @Override
- public void bsp(
- BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, Text> peer)
- throws IOException, SyncException, InterruptedException {
- for (int i = 0; i < 100; i++) {
- // send some messages
- peer.sync();
- }
- }</pre></div>
-
-
- </div></div></div><div class="section"><h3>Shell Command Line Interfaces<a name="Shell_Command_Line_Interfaces"></a></h3></div>
- <p>Hama provides several command for BSP job administration:</p>
-
-<table class="bodyTable" align="center" border="0">
-<tr class="a"><td><b>Command</b></td><td><b>Description</b></td></tr>
-<tr class="b"><td>-submit <job-file></td><td>Submits the job.</td></tr>
-<tr class="a"><td>-status <job-id></td><td>Prints the job status.</td></tr>
-<tr class="b"><td>-kill <job-id></td><td>Kills the job.</td></tr>
-<tr class="a"><td>-list [all]</td><td>-list all displays all jobs. -list displays only jobs which are yet to be completed.</td></tr>
-<tr class="b"><td>-list-active-grooms</td><td>Displays the list of active groom server in the cluster.</td></tr>
-<tr class="a"><td>-list-attempt-ids <jobId> <task-state></td><td>Displays the list of tasks for a given job currently in a particular state (running or completed).</td></tr>
-<tr class="b"><td>-kill-task <task-id></td><td>Kills the task. Killed tasks are NOT counted against failed attempts.</td></tr>
-<tr class="a"><td>-fail-task <task-id></td><td>Fails the task. Failed tasks are counted against failed attempts.</td></tr>
-</table>
-
- <div class="section"><h3>Example: Pi Calculation<a name="Example:_Pi_Calculation"></a></h3></div>
- <p>Here is an BSP-based Pi Calculation example and submit it to Hama cluster:</p>
- <div class="green"><pre>
- private static Path TMP_OUTPUT = new Path("/tmp/pi-" + System.currentTimeMillis());
-
- public static class MyEstimator extends
- BSP<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> {
- public static final Log LOG = LogFactory.getLog(MyEstimator.class);
- private String masterTask;
- private static final int iterations = 10000;
-
- @Override
- public void bsp(
- BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer)
- throws IOException, SyncException, InterruptedException {
-
- int in = 0;
- for (int i = 0; i < iterations; i++) {
- double x = 2.0 * Math.random() - 1.0, y = 2.0 * Math.random() - 1.0;
- if ((Math.sqrt(x * x + y * y) < 1.0)) {
- in++;
- }
- }
-
- double data = 4.0 * in / iterations;
-
- peer.send(masterTask, new DoubleWritable(data));
- peer.sync();
- }
-
- @Override
- public void setup(
- BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer)
- throws IOException {
- // Choose one as a master
- this.masterTask = peer.getPeerName(peer.getNumPeers() / 2);
- }
-
- @Override
- public void cleanup(
- BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer)
- throws IOException {
- if (peer.getPeerName().equals(masterTask)) {
- double pi = 0.0;
- int numPeers = peer.getNumCurrentMessages();
- DoubleWritable received;
- while ((received = peer.getCurrentMessage()) != null) {
- pi += received.get();
- }
-
- pi = pi / numPeers;
- peer.write(new Text("Estimated value of PI is"), new DoubleWritable(pi));
- }
- }
- }
-
- static void printOutput(HamaConfiguration conf) throws IOException {
- FileSystem fs = FileSystem.get(conf);
- FileStatus[] files = fs.listStatus(TMP_OUTPUT);
- for (int i = 0; i < files.length; i++) {
- if (files[i].getLen() > 0) {
- FSDataInputStream in = fs.open(files[i].getPath());
- IOUtils.copyBytes(in, System.out, conf, false);
- in.close();
- break;
- }
- }
-
- fs.delete(TMP_OUTPUT, true);
- }
-
- public static void main(String[] args) throws InterruptedException,
- IOException, ClassNotFoundException {
- // BSP job configuration
- HamaConfiguration conf = new HamaConfiguration();
-
- BSPJob bsp = new BSPJob(conf, PiEstimator.class);
- // Set the job name
- bsp.setJobName("Pi Estimation Example");
- bsp.setBspClass(MyEstimator.class);
- bsp.setInputFormat(NullInputFormat.class);
- bsp.setOutputKeyClass(Text.class);
- bsp.setOutputValueClass(DoubleWritable.class);
- bsp.setOutputFormat(TextOutputFormat.class);
- FileOutputFormat.setOutputPath(bsp, TMP_OUTPUT);
-
- BSPJobClient jobClient = new BSPJobClient(conf);
- ClusterStatus cluster = jobClient.getClusterStatus(true);
-
- if (args.length > 0) {
- bsp.setNumBspTask(Integer.parseInt(args[0]));
- } else {
- // Set to maximum
- bsp.setNumBspTask(cluster.getMaxTasks());
- }
-
- long startTime = System.currentTimeMillis();
- if (bsp.waitForCompletion(true)) {
- printOutput(conf);
- System.out.println("Job Finished in "
- + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
- }
- }</pre></div>
-
-
- </div>
- </div>
- <div class="clear">
- <hr/>
- </div>
- <div id="footer">
- <div class="xright">Copyright © 2008-2012
- <a href="http://www.apache.org/">The Apache Software Foundation</a>.
- All Rights Reserved.
-
- </div>
- <div class="clear">
- <hr/>
- </div>
- </div>
- </body>
-</html>
+ </ul>
+ </div>
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+
+
+ <h3>Information</h3>
+ <ul>
+ <li class="none">
+ <a href="index.html" title="Overview">Overview</a>
+ </li>
+ <li class="none">
+ <a href="downloads.html" title="Downloads">Downloads</a>
+ </li>
+ <li class="none">
+ <a href="team-list.html" title="Who We Are">Who We Are</a>
+ </li>
+ <li class="none">
+ <a href="privacy_policy.html" title="Privacy Policy">Privacy Policy</a>
+ </li>
+ </ul>
+ <h3>Resources</h3>
+ <ul>
+ <li class="none">
+ <a href="developers.html" title="Developers">Developers</a>
+ </li>
+ <li class="none">
+ <a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a>
+ </li>
+ <li class="none">
+ <a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a>
+ </li>
+ <li class="none">
+ <a href="irc-channel.html" title="IRC channel">IRC channel</a>
+ </li>
+ </ul>
+ <h3>Documentation</h3>
+ <ul>
+ <li class="none">
+ <a href="getting_started_with_hama.html" title="Getting Started">Getting Started</a>
+ </li>
+ <li class="none">
+ <a href="hama_on_clouds.html" title="Hama on Clouds">Hama on Clouds</a>
+ </li>
+ <li class="none">
+ <strong>BSP tutorial</strong>
+ </li>
+ <li class="none">
+ <a href="hama_graph_tutorial.html" title="Graph tutorial">Graph tutorial</a>
+ </li>
+ <li class="none">
+ <a href="http://wiki.apache.org/hama/Benchmarks" class="externalLink" title="Benchmarks">Benchmarks</a>
+ </li>
+ <li class="none">
+ <a href="http://wiki.apache.org/hama/Presentations" class="externalLink" title="Presentations">Presentations</a>
+ </li>
+ <li class="none">
+ <a href="http://wiki.apache.org/hama/Articles" class="externalLink" title="Research Papers">Research Papers</a>
+ </li>
+ <li class="none">
+ <a href="http://wiki.apache.org/hama/Books" class="externalLink" title="Books">Books</a>
+ </li>
+ </ul>
+ <h3>Related Projects</h3>
+ <ul>
+ <li class="none">
+ <a href="http://hadoop.apache.org/" class="externalLink" title="Hadoop">Hadoop</a>
+ </li>
+ <li class="none">
+ <a href="http://whirr.apache.org/" class="externalLink" title="Whirr">Whirr</a>
+ </li>
+ <li class="none">
+ <a href="http://incubator.apache.org/bigtop/" class="externalLink" title="Bigtop">Bigtop</a>
+ </li>
+ <li class="none">
+ <a href="http://mahout.apache.org/" class="externalLink" title="Mahout">Mahout</a>
+ </li>
+ </ul>
+ <h3>Art works</h3>
+ <ul>
+ <li class="none">
+ <a href="artwork.html" title="Artwork and Clothing">Artwork and Clothing</a>
+ </li>
+ </ul>
+
+
+
+ <hr class="divider" />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="poweredBy" alt="Built by Maven" src="./images/logos/maven-feather.png" />
+ </a>
+ </div>
+ </div>
+ </div>
+
+ <div id="bodyColumn" class="span10" >
+
+ <!-- Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. -->
+
+ <div class="section"><h2>Hama BSP Tutorial<a name="Hama_BSP_Tutorial"></a></h2></div>
+ <p>This document describes the Hama BSP framework and serves as a tutorial.</p>
+ <div class="section"><h3>Overview<a name="Overview"></a></h3></div>
+ <p>Hama provides a Pure BSP Bulk Synchronous Parallel Model for message passing and collective communication.
+ A BSP program consists of a sequence of supersteps. Each superstep consists of the following three phases:</p>
+
+ <ul>
+ <li>Local computation</li>
+ <li>Process communication</li>
+ <li>Barrier synchronization</li>
+ </ul>
+ <p>BSP programming enables you to write high-performance parallel computing algorithms for a wide range of scientific problems.</p>
+
+ <div class="section"><h3>Create your own BSP by extending BSP class<a name="Create_your_own_BSP_by_extending_BSP_class"></a></h3></div>
+
+ <p>The way to create your own BSP class is to create a class that extends the org.apache.hama.bsp.<b>BSP</b> class.
+ <br />
+ The extending class must override the bsp() method, which is declared like this:
+ </p>
+ <div><pre>
+ public abstract void bsp(BSPPeer<K1, V1, K2, V2, M extends Writable> peer) throws IOException,
+ SyncException, InterruptedException;</pre></div>
+
+ <p>
+ You will define the BSP program inside this bsp() method.
+ It is important to understand that it doesn't mean a single superstep.
+ As described above, a BSP program consists of a sequence of supersteps.
+ So it just gets called once, not all over again unlike Mapper or Reducer method.
+ <br /><br />
+ <b>NOTE</b>: Optionally, there are also setup() and cleanup() which will be called at the beginning of your computation,
+ respectively at the end of the computation. cleanup() is guranteed to run after the computation or in case of failure.
+ You can simply override the methods you need from BSP class.
+ <br /><br />
+ After your own BSP is created, you will need to configure a <b>BSPJob</b> and submit it to Hama cluster to execute a job.
+ The BSP job configuration and submission interfaces is almost the same as the MapReduce job configuration:</p>
+ <div><pre>
+ HamaConfiguration conf = new HamaConfiguration();
+ BSPJob job = new BSPJob(conf, MyBSP.class);
+ job.setJobName("My BSP program");
+ job.setBspClass(MyBSP.class);
+ job.setInputFormat(NullInputFormat.class);
+ job.setOutputKeyClass(Text.class);
+ ...
+ job.waitForCompletion(true);</pre></div>
+ <p>See the below section for more detailed description of BSP user interfaces.</p>
+
+ <div class="section"><h3>User Interfaces<a name="User_Interfaces"></a></h3></div>
+ <div class="section"><div class="section"><h4>Inputs and Outputs<a name="Inputs_and_Outputs"></a></h4>
+ <p>When setting up a BSPJob, you can provide a Input/OutputFormat and Paths like this:</p>
+ <div><pre>
+ job.setInputPath(new Path("/tmp/sequence.dat");
+ job.setInputFormat(org.apache.hama.bsp.SequenceFileInputFormat.class);
+ or,
+ SequenceFileInputFormat.addInputPath(job, new Path("/tmp/sequence.dat"));
+ or,
+ SequenceFileInputFormat.addInputPaths(job, "/tmp/seq1.dat,/tmp/seq2.dat,/tmp/seq3.dat");
+
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+ job.setOutputFormat(TextOutputFormat.class);
+ FileOutputFormat.setOutputPath(job, new Path("/tmp/result"));
+ </pre></div>
+
+ <p>Then, you can read the input and write the output from the methods in BSP class which has "BSPPeer" which contains an communication, counters, and IO interfaces as parameter.
+ In this case we read a normal text file:</p>
+
+ <div><pre>
+ @Override
+ public final void bsp(
+ BSPPeer<LongWritable, Text, Text, LongWritable, Text> peer)
+ throws IOException, InterruptedException, SyncException {
+
+ // this method reads the next key value record from file
+ KeyValuePair<LongWritable, Text> pair = peer.readNext();
+
+ // the following lines do the same:
+ LongWritable key = new LongWritable();
+ Text value = new Text();
+ peer.readNext(key, value);
+
+ // write
+ peer.write(value, key);
+ }</pre></div>
+ <p>Consult the docs for more detail on events like end of file.
+ There is also a function which allows you to re-read the input from the beginning.
+ This snippet reads the input five times:
+ </p>
+ <div><pre>
+ for(int i = 0; i < 5; i++){
+ LongWritable key = new LongWritable();
+ Text value = new Text();
+ while (peer.readNext(key, value)) {
+ // read everything
+ }
+ // reopens the input
+ peer.reopenInput()
+ }</pre></div>
+
+ </div><div class="section"><h4>Communication<a name="Communication"></a></h4>
+ <p>Hama BSP provides simple but powerful communication APIs for many purposes.
+ We tried to follow the standard library of BSP world as much as possible.
+ The following table describes all the methods you can use:</p>
+
+<table class="table table-striped" align="center" border="0">
+<tr class="a"><td><b>Method</b></td><td><b>Description</b></td></tr>
+<tr class="b"><td>send(String peerName, BSPMessage msg)</td><td>Sends a message to another peer.</td></tr>
+<tr class="a"><td>getCurrentMessage()</td><td>Returns a received message.</td></tr>
+<tr class="b"><td>getNumCurrentMessages()</td><td>Returns the number of received messages.</td></tr>
+<tr class="a"><td>sync()</td><td>Barrier synchronization.</td></tr>
+<tr class="b"><td>getPeerName()</td><td>Returns a peer's hostname.</td></tr>
+<tr class="a"><td>getAllPeerNames()</td><td>Returns all peer's hostname.</td></tr>
+<tr class="b"><td>getSuperstepCount()</td><td>Returns the count of supersteps</td></tr>
+</table>
+
+ <p>The send() and all the other functions are very flexible. Here is an example that sends a message to all peers:</p>
+
+ <div><pre>
+ @Override
+ public void bsp(
+ BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, Text> peer)
+ throws IOException, SyncException, InterruptedException {
+ for (String peerName : peer.getAllPeerNames()) {
+ peer.send(peerName,
+ new Text("Hello from " + peer.getPeerName(), System.currentTimeMillis()));
+ }
+
+ peer.sync();
+ }</pre></div>
+
+ </div><div class="section"><h4>Synchronization<a name="Synchronization"></a></h4>
+
+ <p>When all the processes have entered the barrier via the sync() method,
+ the Hama proceeds to the next superstep.
+ In the previous example, the BSP job will be finished by one synchronization
+ after sending a message "Hello from ..." to all peers.
+ <br /><br />
+ But, keep in mind that the sync() function is not the end of the BSP job.
+ As was previously mentioned, all the communication functions are very flexible.
+ For example, the sync() method also can be called in a for loop
+ so that you can use to program the iterative methods sequentially:</p>
+
+ <div><pre>
+ @Override
+ public void bsp(
+ BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, Text> peer)
+ throws IOException, SyncException, InterruptedException {
+ for (int i = 0; i < 100; i++) {
+ // send some messages
+ peer.sync();
+ }
+ }</pre></div>
+
+
+ </div></div><div class="section"><h3>Shell Command Line Interfaces<a name="Shell_Command_Line_Interfaces"></a></h3></div>
+ <p>Hama provides several command for BSP job administration:</p>
+
+<table class="table table-striped" align="center" border="0">
+<tr class="a"><td><b>Command</b></td><td><b>Description</b></td></tr>
+<tr class="b"><td>-submit <job-file></td><td>Submits the job.</td></tr>
+<tr class="a"><td>-status <job-id></td><td>Prints the job status.</td></tr>
+<tr class="b"><td>-kill <job-id></td><td>Kills the job.</td></tr>
+<tr class="a"><td>-list [all]</td><td>-list all displays all jobs. -list displays only jobs which are yet to be completed.</td></tr>
+<tr class="b"><td>-list-active-grooms</td><td>Displays the list of active groom server in the cluster.</td></tr>
+<tr class="a"><td>-list-attempt-ids <jobId> <task-state></td><td>Displays the list of tasks for a given job currently in a particular state (running or completed).</td></tr>
+<tr class="b"><td>-kill-task <task-id></td><td>Kills the task. Killed tasks are NOT counted against failed attempts.</td></tr>
+<tr class="a"><td>-fail-task <task-id></td><td>Fails the task. Failed tasks are counted against failed attempts.</td></tr>
+</table>
+
+ <div class="section"><h3>Example: Pi Calculation<a name="Example:_Pi_Calculation"></a></h3></div>
+ <p>Here is an BSP-based Pi Calculation example and submit it to Hama cluster:</p>
+ <div><pre>
+ private static Path TMP_OUTPUT = new Path("/tmp/pi-" + System.currentTimeMillis());
+
+ public static class MyEstimator extends
+ BSP<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> {
+ public static final Log LOG = LogFactory.getLog(MyEstimator.class);
+ private String masterTask;
+ private static final int iterations = 10000;
+
+ @Override
+ public void bsp(
+ BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer)
+ throws IOException, SyncException, InterruptedException {
+
+ int in = 0;
+ for (int i = 0; i < iterations; i++) {
+ double x = 2.0 * Math.random() - 1.0, y = 2.0 * Math.random() - 1.0;
+ if ((Math.sqrt(x * x + y * y) < 1.0)) {
+ in++;
+ }
+ }
+
+ double data = 4.0 * in / iterations;
+
+ peer.send(masterTask, new DoubleWritable(data));
+ peer.sync();
+ }
+
+ @Override
+ public void setup(
+ BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer)
+ throws IOException {
+ // Choose one as a master
+ this.masterTask = peer.getPeerName(peer.getNumPeers() / 2);
+ }
+
+ @Override
+ public void cleanup(
+ BSPPeer<NullWritable, NullWritable, Text, DoubleWritable, DoubleWritable> peer)
+ throws IOException {
+ if (peer.getPeerName().equals(masterTask)) {
+ double pi = 0.0;
+ int numPeers = peer.getNumCurrentMessages();
+ DoubleWritable received;
+ while ((received = peer.getCurrentMessage()) != null) {
+ pi += received.get();
+ }
+
+ pi = pi / numPeers;
+ peer.write(new Text("Estimated value of PI is"), new DoubleWritable(pi));
+ }
+ }
+ }
+
+ static void printOutput(HamaConfiguration conf) throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ FileStatus[] files = fs.listStatus(TMP_OUTPUT);
+ for (int i = 0; i < files.length; i++) {
+ if (files[i].getLen() > 0) {
+ FSDataInputStream in = fs.open(files[i].getPath());
+ IOUtils.copyBytes(in, System.out, conf, false);
+ in.close();
+ break;
+ }
+ }
+
+ fs.delete(TMP_OUTPUT, true);
+ }
+
+ public static void main(String[] args) throws InterruptedException,
+ IOException, ClassNotFoundException {
+ // BSP job configuration
+ HamaConfiguration conf = new HamaConfiguration();
+
+ BSPJob bsp = new BSPJob(conf, PiEstimator.class);
+ // Set the job name
+ bsp.setJobName("Pi Estimation Example");
+ bsp.setBspClass(MyEstimator.class);
+ bsp.setInputFormat(NullInputFormat.class);
+ bsp.setOutputKeyClass(Text.class);
+ bsp.setOutputValueClass(DoubleWritable.class);
+ bsp.setOutputFormat(TextOutputFormat.class);
+ FileOutputFormat.setOutputPath(bsp, TMP_OUTPUT);
+
+ BSPJobClient jobClient = new BSPJobClient(conf);
+ ClusterStatus cluster = jobClient.getClusterStatus(true);
+
+ if (args.length > 0) {
+ bsp.setNumBspTask(Integer.parseInt(args[0]));
+ } else {
+ // Set to maximum
+ bsp.setNumBspTask(cluster.getMaxTasks());
+ }
+
+ long startTime = System.currentTimeMillis();
+ if (bsp.waitForCompletion(true)) {
+ printOutput(conf);
+ System.out.println("Job Finished in "
+ + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
+ }
+ }</pre></div>
+
+
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row span12">Copyright © 2008-2012
+ <a href="http://www.apache.org/">The Apache Software Foundation</a>.
+ All Rights Reserved.
+
+ </div>
+
+
+
+ </div>
+ </footer>
+ </body>
+</html>