You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2012/02/11 11:22:31 UTC

svn commit: r1243022 [30/38] - in /mahout/site/new_website: ./ MAHOUT/ MAHOUT/2010/ MAHOUT/2010/09/ MAHOUT/2010/09/14/ MAHOUT/2011/ MAHOUT/2011/10/ MAHOUT/2011/10/21/ MAHOUT/books-tutorials-and-talks.data/ MAHOUT/books-tutorials-talks.data/ MAHOUT/book...

Added: mahout/site/new_website/MAHOUT/top-down-clustering.html
URL: http://svn.apache.org/viewvc/mahout/site/new_website/MAHOUT/top-down-clustering.html?rev=1243022&view=auto
==============================================================================
--- mahout/site/new_website/MAHOUT/top-down-clustering.html (added)
+++ mahout/site/new_website/MAHOUT/top-down-clustering.html Sat Feb 11 10:22:15 2012
@@ -0,0 +1,213 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<HTML>
+  <HEAD>
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/space.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/master.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/wiki-content.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/abs.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/menu.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/menu-ie.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/tables.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/panels.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/master-ie.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/renderer-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/content-types.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/login.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/information-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/layout-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/default-theme.css">
+    <LINK type="text/css" rel="stylesheet" href="resources/space.css">
+    <STYLE type="text/css">
+      .footer {
+        background-image:      url('https://cwiki.apache.org/confluence/images/border/border_bottom.gif');
+        background-repeat:     repeat-x;
+        background-position:   left top;
+        padding-top:           4px;
+        color:                 #666;
+      }
+    </STYLE>
+    <SCRIPT type="text/javascript" language="javascript">
+      var hide = null;
+      var show = null;
+      var children = null;
+
+      function init() {
+        /* Search form initialization */
+        var form = document.forms['search'];
+        if (form != null) {
+          form.elements['domains'].value = location.hostname;
+          form.elements['sitesearch'].value = location.hostname;
+        }
+
+        /* Children initialization */
+        hide = document.getElementById('hide');
+        show = document.getElementById('show');
+        children = document.all != null ?
+                   document.all['children'] :
+                   document.getElementById('children');
+        if (children != null) {
+          children.style.display = 'none';
+          show.style.display = 'inline';
+          hide.style.display = 'none';
+        }
+      }
+
+      function showChildren() {
+        children.style.display = 'block';
+        show.style.display = 'none';
+        hide.style.display = 'inline';
+      }
+
+      function hideChildren() {
+        children.style.display = 'none';
+        show.style.display = 'inline';
+        hide.style.display = 'none';
+      }
+    </SCRIPT>
+    <TITLE>Top Down Clustering</TITLE>
+  <META http-equiv="Content-Type" content="text/html;charset=UTF-8"></HEAD>
+  <BODY onload="init()">
+    <TABLE border="0" cellpadding="2" cellspacing="0" width="100%">
+      <TR class="topBar">
+        <TD align="left" valign="middle" class="topBarDiv" align="left" nowrap="">
+          &nbsp;<A href="mahout-wiki.html" title="Apache Mahout">Apache Mahout</A>&nbsp;&gt;&nbsp;<A href="mahout-wiki.html" title="Mahout Wiki">Mahout Wiki</A>&nbsp;&gt;&nbsp;<A href="algorithms.html" title="Algorithms">Algorithms</A>&nbsp;&gt;&nbsp;<A href="" title="Top Down Clustering">Top Down Clustering</A>
+        </TD>
+        <TD align="right" valign="middle" nowrap="">
+          <FORM name="search" action="http://www.google.com/search" method="get">
+            <INPUT type="hidden" name="ie" value="UTF-8">
+            <INPUT type="hidden" name="oe" value="UTF-8">
+            <INPUT type="hidden" name="domains" value="">
+            <INPUT type="hidden" name="sitesearch" value="">
+            <INPUT type="text" name="q" maxlength="255" value="">        
+            <INPUT type="submit" name="btnG" value="Google Search">
+          </FORM>
+        </TD>
+      </TR> 
+    </TABLE>
+
+    <DIV id="PageContent">
+      <DIV class="pageheader" style="padding: 6px 0px 0px 0px;">
+        <!-- We'll enable this once we figure out how to access (and save) the logo resource -->
+        <!--img src="/wiki/images/confluence_logo.gif" style="float: left; margin: 4px 4px 4px 10px;" border="0"-->
+        <DIV style="margin: 0px 10px 0px 10px" class="smalltext">Apache Mahout</DIV>
+        <DIV style="margin: 0px 10px 8px 10px" class="pagetitle">Top Down Clustering</DIV>
+
+        <DIV class="greynavbar" align="right" style="padding: 2px 10px; margin: 0px;">
+          <A href="https://cwiki.apache.org/confluence/pages/editpage.action?pageId=27832740">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/notep_16.gif" height="16" width="16" border="0" align="absmiddle" title="Edit Page"></A>
+            <A href="https://cwiki.apache.org/confluence/pages/editpage.action?pageId=27832740">Edit Page</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/listpages.action?key=MAHOUT">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/browse_space.gif" height="16" width="16" border="0" align="absmiddle" title="Browse Space"></A>
+            <A href="https://cwiki.apache.org/confluence/pages/listpages.action?key=MAHOUT">Browse Space</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/createpage.action?spaceKey=MAHOUT&fromPageId=27832740">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/add_page_16.gif" height="16" width="16" border="0" align="absmiddle" title="Add Page"></A>
+          <A href="https://cwiki.apache.org/confluence/pages/createpage.action?spaceKey=MAHOUT&fromPageId=27832740">Add Page</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/createblogpost.action?spaceKey=MAHOUT&fromPageId=27832740">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/add_blogentry_16.gif" height="16" width="16" border="0" align="absmiddle" title="Add News"></A>
+          <A href="https://cwiki.apache.org/confluence/pages/createblogpost.action?spaceKey=MAHOUT&fromPageId=27832740">Add News</A>
+        </DIV>
+      </DIV>
+
+      <DIV class="pagecontent">
+        <DIV class="wiki-content">
+          <H2><A name="TopDownClustering-TopDownClustering"></A>Top Down Clustering</H2>
+
+<P>Top Down clustering is a type of Hierarchical Clustering. It tries to find bigger clusters first and then does fine grained clustering on these clusters. Hence the name Top Down.</P>
+
+<P>Any clustering algorithm can be used to perform the Top Level Clustering ( finding bigger clusters ) and the Bottom Level Clustering ( fine grained clustering on each of the top level clusters). So, all clustering algorithms available in Mahout, other than the MinHash Clustering algorithm ( which is a &quot;Bottom Up&quot; Clustering Algorithm ), are suitable to be used for Top Down Clustering, on both Top Level and Bottom Level.</P>
+
+<P>The top level clustering output needs to be post processed in order to identify all top level clusters and, to group vectors into their respective top level clusters. So, that, the bottom level clustering can execute on each of them.</P>
+
+<P>The first step to execute the top down clustering, would be to run any clustering algorithm of your choice, preferably with clustering parameters which will produce bigger clusters. This would be the top level clustering.</P>
+
+<P>Then, the output of this clustering should be post processed, to group them into respective top level clusters. This can be done using <B>ClusterOutputPostProcessorDriver.</B></P>
+
+<H2><A name="TopDownClustering-Designofimplementation"></A>Design of implementation</H2>
+
+<P>When any clustering algorithm runs, the output path stores data in two directories</P>
+
+<P><B>clusteredPoints</B></P>
+
+<P><B>clusters-0-final</B></P>
+
+<P>The clusteredPoints directory contains&nbsp;information in the form of <EM>(clusterId,</EM> <B><EM>vector)</EM></B><B>.</B></P>
+
+<P>The clusters-*-final directory will hold the cluster centroids.</P>
+
+<P>Now, to further run clustering on the clusters found, the vectors belonging to different clusters needs to be stored in separate directories. This can be done using the <B>ClusterOutputPostProcessorDriver</B> as explained in the <B><EM>Usage</EM></B> section*.&#42;</P>
+
+<P><B>ClusterOutputPostProcessorDriver</B> will need this output path as the input, and it will segregate it into separate clusters.</P>
+
+<P>So, after post processing, if you will check the output path provided to the <B>ClusterOutputPostProcessorDriver,</B> you will find directories with names as clusterId, i.e. 0,1,2,...,20,21,22,23,24.25....</P>
+
+<P>All these directories will store files containing the vectors for that particular cluster. Now, all of these directories can be provided as input to the bottom level clustering algorithm one by one. The bottom level clustering algorithm can then, cluster all the top level clusters as per the algorithm used.</P>
+
+<H2><A name="TopDownClustering-Running"></A>Running</H2>
+
+<H3><A name="TopDownClustering-JavaAPI"></A>Java API</H3>
+
+<P><B>ClusterOutputPostProcessorDriver</B> has a run method</P>
+
+<P><B>run(Path input, Path output, boolean runSequential)</B></P>
+
+<P>The input parameter provided to run method is, <EM>&quot;the output path provided to the clustering algorithm&quot;</EM>, which would be post processed. It is the path of the directory containing clusters-*-final and clusteredPoints.</P>
+
+<P>The output parameter provided to run method is <EM>&quot;the path where the post processed data would be stored&quot;</EM>.</P>
+
+<P>The runSequential parameter provided to run method is <EM>&quot;If set to true, post processes it sequentially, else, uses, MapReduce to do it&quot;</EM>. Hint : If the clustering was done sequentially, make it sequential, else vice versa.</P>
+
+<H3><A name="TopDownClustering-CommandLineInvocation"></A>Command Line Invocation</H3>
+
+<P>The following script illustrates the use of the CLI interface to run a top-down clustering example based upon the Synthetic Control dataset. This example uses K-means to cluster the data set. Other algorithms can also be used as described above. To use this script, download the dataset into a &lt;testdata&gt; directory or run the examples/bin/cluster-syntheticcontrol.sh.</P>
+
+<DIV class="code panel" style="border-width: 1px;"><DIV class="codeContent panelContent">
+<PRE class="code-java">
+unset HADOOP_HOME
+unset HADOOP_CONF_DIR
+rm -rf data top bottom
+./bin/mahout org.apache.mahout.clustering.conversion.InputDriver -i &lt;testdata&gt; -o data
+./bin/mahout kmeans -i data -o top -c top/clusters-0 -k 5 -xm sequential -ow -cl -x 5
+./bin/mahout clusterdump -s top/clusters-*-<SPAN class="code-keyword">final</SPAN>/
+mkdir bottom
+./bin/mahout clusterpp -i top -o bottom/data -xm sequential
+<SPAN class="code-keyword">for</SPAN> x in `ls bottom/data $1`; <SPAN class="code-keyword">do</SPAN>
+echo
+./bin/mahout kmeans -i bottom/data/$x -o bottom/$x -c bottom/$x/clusters-0 -k 5 -xm sequential -ow -cl -x 5;
+./bin/mahout clusterdump -s bottom/$x/clusters-*-<SPAN class="code-keyword">final</SPAN>/;
+done
+</PRE>
+</DIV></DIV>
+
+
+<P><SPAN class="image-wrap" style="display: block; text-align: center"><IMG src="top-down-clustering.data/Top%20Down%20Clustering.jpg" style="border: 1px solid black"></SPAN></P>
+        </DIV>
+
+        
+      </DIV>
+    </DIV>
+    <DIV class="footer">
+      Generated by
+      <A href="http://www.atlassian.com/confluence/">Atlassian Confluence</A> (Version: 3.4.9 Build: 2042 Feb 14, 2011)
+      <A href="http://could.it/autoexport/">Auto Export Plugin</A> (Version: 1.0.0-dkulp)
+    </DIV>
+<SCRIPT type="text/javascript">
+
+  var _gaq = _gaq || [];
+  _gaq.push(['_setAccount', 'UA-17359171-1']);
+  _gaq.push(['_setDomainName', 'none']);
+  _gaq.push(['_setAllowLinker', true]);
+  _gaq.push(['_trackPageview']);
+
+  (function() {
+    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+  })();
+
+</SCRIPT>
+  </BODY>
+</HTML>
\ No newline at end of file

Added: mahout/site/new_website/MAHOUT/traveling-salesman.html
URL: http://svn.apache.org/viewvc/mahout/site/new_website/MAHOUT/traveling-salesman.html?rev=1243022&view=auto
==============================================================================
--- mahout/site/new_website/MAHOUT/traveling-salesman.html (added)
+++ mahout/site/new_website/MAHOUT/traveling-salesman.html Sat Feb 11 10:22:15 2012
@@ -0,0 +1,162 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<HTML>
+  <HEAD>
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/space.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/master.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/wiki-content.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/abs.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/menu.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/menu-ie.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/tables.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/panels.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/master-ie.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/renderer-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/content-types.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/login.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/information-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/layout-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/default-theme.css">
+    <LINK type="text/css" rel="stylesheet" href="resources/space.css">
+    <STYLE type="text/css">
+      .footer {
+        background-image:      url('https://cwiki.apache.org/confluence/images/border/border_bottom.gif');
+        background-repeat:     repeat-x;
+        background-position:   left top;
+        padding-top:           4px;
+        color:                 #666;
+      }
+    </STYLE>
+    <SCRIPT type="text/javascript" language="javascript">
+      var hide = null;
+      var show = null;
+      var children = null;
+
+      function init() {
+        /* Search form initialization */
+        var form = document.forms['search'];
+        if (form != null) {
+          form.elements['domains'].value = location.hostname;
+          form.elements['sitesearch'].value = location.hostname;
+        }
+
+        /* Children initialization */
+        hide = document.getElementById('hide');
+        show = document.getElementById('show');
+        children = document.all != null ?
+                   document.all['children'] :
+                   document.getElementById('children');
+        if (children != null) {
+          children.style.display = 'none';
+          show.style.display = 'inline';
+          hide.style.display = 'none';
+        }
+      }
+
+      function showChildren() {
+        children.style.display = 'block';
+        show.style.display = 'none';
+        hide.style.display = 'inline';
+      }
+
+      function hideChildren() {
+        children.style.display = 'none';
+        show.style.display = 'inline';
+        hide.style.display = 'none';
+      }
+    </SCRIPT>
+    <TITLE>Traveling Salesman</TITLE>
+  <META http-equiv="Content-Type" content="text/html;charset=UTF-8"></HEAD>
+  <BODY onload="init()">
+    <TABLE border="0" cellpadding="2" cellspacing="0" width="100%">
+      <TR class="topBar">
+        <TD align="left" valign="middle" class="topBarDiv" align="left" nowrap="">
+          &nbsp;<A href="mahout-wiki.html" title="Apache Mahout">Apache Mahout</A>&nbsp;&gt;&nbsp;<A href="mahout-wiki.html" title="Mahout Wiki">Mahout Wiki</A>&nbsp;&gt;&nbsp;<A href="quickstart.html" title="Quickstart">Quickstart</A>&nbsp;&gt;&nbsp;<A href="" title="Traveling Salesman">Traveling Salesman</A>
+        </TD>
+        <TD align="right" valign="middle" nowrap="">
+          <FORM name="search" action="http://www.google.com/search" method="get">
+            <INPUT type="hidden" name="ie" value="UTF-8">
+            <INPUT type="hidden" name="oe" value="UTF-8">
+            <INPUT type="hidden" name="domains" value="">
+            <INPUT type="hidden" name="sitesearch" value="">
+            <INPUT type="text" name="q" maxlength="255" value="">        
+            <INPUT type="submit" name="btnG" value="Google Search">
+          </FORM>
+        </TD>
+      </TR> 
+    </TABLE>
+
+    <DIV id="PageContent">
+      <DIV class="pageheader" style="padding: 6px 0px 0px 0px;">
+        <!-- We'll enable this once we figure out how to access (and save) the logo resource -->
+        <!--img src="/wiki/images/confluence_logo.gif" style="float: left; margin: 4px 4px 4px 10px;" border="0"-->
+        <DIV style="margin: 0px 10px 0px 10px" class="smalltext">Apache Mahout</DIV>
+        <DIV style="margin: 0px 10px 8px 10px" class="pagetitle">Traveling Salesman</DIV>
+
+        <DIV class="greynavbar" align="right" style="padding: 2px 10px; margin: 0px;">
+          <A href="https://cwiki.apache.org/confluence/pages/editpage.action?pageId=100780">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/notep_16.gif" height="16" width="16" border="0" align="absmiddle" title="Edit Page"></A>
+            <A href="https://cwiki.apache.org/confluence/pages/editpage.action?pageId=100780">Edit Page</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/listpages.action?key=MAHOUT">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/browse_space.gif" height="16" width="16" border="0" align="absmiddle" title="Browse Space"></A>
+            <A href="https://cwiki.apache.org/confluence/pages/listpages.action?key=MAHOUT">Browse Space</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/createpage.action?spaceKey=MAHOUT&fromPageId=100780">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/add_page_16.gif" height="16" width="16" border="0" align="absmiddle" title="Add Page"></A>
+          <A href="https://cwiki.apache.org/confluence/pages/createpage.action?spaceKey=MAHOUT&fromPageId=100780">Add Page</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/createblogpost.action?spaceKey=MAHOUT&fromPageId=100780">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/add_blogentry_16.gif" height="16" width="16" border="0" align="absmiddle" title="Add News"></A>
+          <A href="https://cwiki.apache.org/confluence/pages/createblogpost.action?spaceKey=MAHOUT&fromPageId=100780">Add News</A>
+        </DIV>
+      </DIV>
+
+      <DIV class="pagecontent">
+        <DIV class="wiki-content">
+          <H1><A name="TravelingSalesman-Intro"></A>Intro</H1>
+
+<P>The Traveling Salesman Problem (TSP) is a classic computer science question classified as NP-Hard.  See <A href="http://en.wikipedia.org/wiki/Travelling_salesman_problem" class="external-link" rel="nofollow">http://en.wikipedia.org/wiki/Travelling_salesman_problem</A> for background information.</P>
+
+<H1><A name="TravelingSalesman-EvolutionaryExample"></A>Evolutionary Example</H1>
+
+<P>As an example of evolutionary programming, Mahout has an example implementation that attempts to solve TSP.  It should be noted that the implementation of evolutionary programming in Mahout has not been maintained for some time and may be removed in the future due to lack of interest.</P>
+
+<P>To run the example, do:</P>
+
+<OL>
+	<LI>cd &lt;MAHOUT_HOME&gt;</LI>
+	<LI>mvn install</LI>
+	<LI>Run the Job: 
+<DIV class="code panel" style="border-width: 1px;"><DIV class="codeContent panelContent">
+<PRE class="code-java">./bin/mahout org.apache.mahout.ga.watchmaker.travellingsalesman.TravellingSalesman</PRE>
+</DIV></DIV></LI>
+</OL>
+
+        </DIV>
+
+        
+      </DIV>
+    </DIV>
+    <DIV class="footer">
+      Generated by
+      <A href="http://www.atlassian.com/confluence/">Atlassian Confluence</A> (Version: 3.4.9 Build: 2042 Feb 14, 2011)
+      <A href="http://could.it/autoexport/">Auto Export Plugin</A> (Version: 1.0.0-dkulp)
+    </DIV>
+<SCRIPT type="text/javascript">
+
+  var _gaq = _gaq || [];
+  _gaq.push(['_setAccount', 'UA-17359171-1']);
+  _gaq.push(['_setDomainName', 'none']);
+  _gaq.push(['_setAllowLinker', true]);
+  _gaq.push(['_trackPageview']);
+
+  (function() {
+    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+  })();
+
+</SCRIPT>
+  </BODY>
+</HTML>
\ No newline at end of file

Added: mahout/site/new_website/MAHOUT/twenty-newsgroups.html
URL: http://svn.apache.org/viewvc/mahout/site/new_website/MAHOUT/twenty-newsgroups.html?rev=1243022&view=auto
==============================================================================
--- mahout/site/new_website/MAHOUT/twenty-newsgroups.html (added)
+++ mahout/site/new_website/MAHOUT/twenty-newsgroups.html Sat Feb 11 10:22:15 2012
@@ -0,0 +1,269 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<HTML>
+  <HEAD>
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/space.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/master.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/wiki-content.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/abs.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/menu.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/menu-ie.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/tables.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/panels.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/master-ie.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/renderer-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/content-types.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/login.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/information-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/layout-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/default-theme.css">
+    <LINK type="text/css" rel="stylesheet" href="resources/space.css">
+    <STYLE type="text/css">
+      .footer {
+        background-image:      url('https://cwiki.apache.org/confluence/images/border/border_bottom.gif');
+        background-repeat:     repeat-x;
+        background-position:   left top;
+        padding-top:           4px;
+        color:                 #666;
+      }
+    </STYLE>
+    <SCRIPT type="text/javascript" language="javascript">
+      var hide = null;
+      var show = null;
+      var children = null;
+
+      function init() {
+        /* Search form initialization */
+        var form = document.forms['search'];
+        if (form != null) {
+          form.elements['domains'].value = location.hostname;
+          form.elements['sitesearch'].value = location.hostname;
+        }
+
+        /* Children initialization */
+        hide = document.getElementById('hide');
+        show = document.getElementById('show');
+        children = document.all != null ?
+                   document.all['children'] :
+                   document.getElementById('children');
+        if (children != null) {
+          children.style.display = 'none';
+          show.style.display = 'inline';
+          hide.style.display = 'none';
+        }
+      }
+
+      function showChildren() {
+        children.style.display = 'block';
+        show.style.display = 'none';
+        hide.style.display = 'inline';
+      }
+
+      function hideChildren() {
+        children.style.display = 'none';
+        show.style.display = 'inline';
+        hide.style.display = 'none';
+      }
+    </SCRIPT>
+    <TITLE>Twenty Newsgroups</TITLE>
+  <META http-equiv="Content-Type" content="text/html;charset=UTF-8"></HEAD>
+  <BODY onload="init()">
+    <TABLE border="0" cellpadding="2" cellspacing="0" width="100%">
+      <TR class="topBar">
+        <TD align="left" valign="middle" class="topBarDiv" align="left" nowrap="">
+          &nbsp;<A href="mahout-wiki.html" title="Apache Mahout">Apache Mahout</A>&nbsp;&gt;&nbsp;<A href="mahout-wiki.html" title="Mahout Wiki">Mahout Wiki</A>&nbsp;&gt;&nbsp;<A href="quickstart.html" title="Quickstart">Quickstart</A>&nbsp;&gt;&nbsp;<A href="" title="Twenty Newsgroups">Twenty Newsgroups</A>
+        </TD>
+        <TD align="right" valign="middle" nowrap="">
+          <FORM name="search" action="http://www.google.com/search" method="get">
+            <INPUT type="hidden" name="ie" value="UTF-8">
+            <INPUT type="hidden" name="oe" value="UTF-8">
+            <INPUT type="hidden" name="domains" value="">
+            <INPUT type="hidden" name="sitesearch" value="">
+            <INPUT type="text" name="q" maxlength="255" value="">        
+            <INPUT type="submit" name="btnG" value="Google Search">
+          </FORM>
+        </TD>
+      </TR> 
+    </TABLE>
+
+    <DIV id="PageContent">
+      <DIV class="pageheader" style="padding: 6px 0px 0px 0px;">
+        <!-- We'll enable this once we figure out how to access (and save) the logo resource -->
+        <!--img src="/wiki/images/confluence_logo.gif" style="float: left; margin: 4px 4px 4px 10px;" border="0"-->
+        <DIV style="margin: 0px 10px 0px 10px" class="smalltext">Apache Mahout</DIV>
+        <DIV style="margin: 0px 10px 8px 10px" class="pagetitle">Twenty Newsgroups</DIV>
+
+        <DIV class="greynavbar" align="right" style="padding: 2px 10px; margin: 0px;">
+          <A href="https://cwiki.apache.org/confluence/pages/editpage.action?pageId=99739">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/notep_16.gif" height="16" width="16" border="0" align="absmiddle" title="Edit Page"></A>
+            <A href="https://cwiki.apache.org/confluence/pages/editpage.action?pageId=99739">Edit Page</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/listpages.action?key=MAHOUT">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/browse_space.gif" height="16" width="16" border="0" align="absmiddle" title="Browse Space"></A>
+            <A href="https://cwiki.apache.org/confluence/pages/listpages.action?key=MAHOUT">Browse Space</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/createpage.action?spaceKey=MAHOUT&fromPageId=99739">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/add_page_16.gif" height="16" width="16" border="0" align="absmiddle" title="Add Page"></A>
+          <A href="https://cwiki.apache.org/confluence/pages/createpage.action?spaceKey=MAHOUT&fromPageId=99739">Add Page</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/createblogpost.action?spaceKey=MAHOUT&fromPageId=99739">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/add_blogentry_16.gif" height="16" width="16" border="0" align="absmiddle" title="Add News"></A>
+          <A href="https://cwiki.apache.org/confluence/pages/createblogpost.action?spaceKey=MAHOUT&fromPageId=99739">Add News</A>
+        </DIV>
+      </DIV>
+
+      <DIV class="pagecontent">
+        <DIV class="wiki-content">
+          <H2><A name="TwentyNewsgroups-TwentyNewsgroupsClassificationExample"></A>Twenty Newsgroups Classification Example</H2>
+
+<H2><A name="TwentyNewsgroups-Introduction"></A>Introduction</H2>
+
+<P>The 20 Newsgroups data set is a collection of approximately 20,000 newsgroup documents, partitioned (nearly) evenly across 20 different newsgroups. The 20 newsgroups collection has become a popular data set for experiments in text applications of machine learning techniques, such as text classification and text clustering. We will use Mahout Bayes Classifier to create a model that would classify a new document into one of the 20 newsgroup.</P>
+
+<H2><A name="TwentyNewsgroups-Prerequisites"></A>Prerequisites</H2>
+
+<UL>
+	<LI>Mahout has been downloaded (<A href="http://cwiki.apache.org/confluence/display/MAHOUT/index#index-Installation%252FSetup" class="external-link" rel="nofollow">instructions here</A>)</LI>
+	<LI>Maven is available</LI>
+	<LI>Your environment has the following variables:
+<DIV class="table-wrap">
+<TABLE class="confluenceTable"><TBODY>
+<TR>
+<TD class="confluenceTd"> <TT>HADOOP_HOME</TT> </TD>
+<TD class="confluenceTd"> Environment variables refers to where Hadoop lives </TD>
+</TR>
+<TR>
+<TD class="confluenceTd"> <TT>MAHOUT_HOME</TT> </TD>
+<TD class="confluenceTd"> Environment variables refers to where Mahout lives </TD>
+</TR>
+</TBODY></TABLE>
+</DIV>
+</LI>
+</UL>
+
+
+<H2><A name="TwentyNewsgroups-Instructionsforrunningtheexample"></A>Instructions for running the example</H2>
+
+<OL>
+	<LI>Start the hadoop daemons by executing the following commands
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ cd $HADOOP_HOME/bin
+$ ./start-all.sh
+</PRE>
+</DIV></DIV></LI>
+	<LI>In the trunk directory of mahout, compile everything and create the mahout job:
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ cd $MAHOUT_HOME
+$ mvn install
+</PRE>
+</DIV></DIV></LI>
+	<LI>Run the 20 newsgroup example by executing the script as below
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ ./examples/bin/build-20news-bayes.sh
+</PRE>
+</DIV></DIV>
+<P>After MAHOUT-857 is committed (available when 0.6 is released), the command will be:</P>
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ ./examples/bin/classify-20newsgroups.sh
+</PRE>
+</DIV></DIV>
+<P>This later version allows you to also try out running Stochastic Gradient Descent (SGD) on the same data.</P></LI>
+</OL>
+
+
+<P>The script performs the following</P>
+<OL>
+	<LI>
+	<OL>
+		<LI>Downloads <TT>20news-bydate.tar.gz</TT> from the <A href="http://people.csail.mit.edu/jrennie/20Newsgroups/20news-bydate.tar.gz" class="external-link" rel="nofollow">20newsgroups dataset</A></LI>
+		<LI>Extracts dataset</LI>
+		<LI>Generates input dataset for training classifier</LI>
+		<LI>Generates input dataset for testing classifier</LI>
+		<LI>Trains the classifier</LI>
+		<LI>Tests the classifier</LI>
+	</OL>
+	</LI>
+</OL>
+
+
+<P>Output might look like:</P>
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>=======================================================
+Confusion Matrix
+-------------------------------------------------------
+a   b   c   d   e   f   g   h   i   j   k   l   m   n   o   p   q   r   s   t   u   &lt;--Classified as
+381 0   0   0   0   9   1   0   0   0   1   0   0   2   0   1   0   0   3   0   0    |  398  a     = rec.motorcycles
+1   284 0   0   0   0   1   0   6   3   11  0   66  3   0   1   6   0   4   9   0    |  395  b     = comp.windows.x
+2   0   339 2   0   3   5   1   0   0   0   0   1   1   12  1   7   0   2   0   0    |  376  c     = talk.politics.mideast
+4   0   1   327 0   2   2   0   0   2   1   1   0   5   1   4   12  0   2   0   0    |  364  d     = talk.politics.guns
+7   0   4   32  27  7   7   2   0   12  0   0   6   0   100 9   7   31  0   0   0    |  251  e     = talk.religion.misc
+10  0   0   0   0   359 2   2   0   1   3   0   1   6   0   1   0   0   11  0   0    |  396  f     = rec.autos
+0   0   0   0   0   1   383 9   1   0   0   0   0   0   0   0   0   0   3   0   0    |  397  g     = rec.sport.baseball
+1   0   0   0   0   0   9   382 0   0   0   0   1   1   1   0   2   0   2   0   0    |  399  h     = rec.sport.hockey
+2   0   0   0   0   4   3   0   330 4   4   0   5   12  0   0   2   0   12  7   0    |  385  i     = comp.sys.mac.hardware
+0   3   0   0   0   0   1   0   0   368 0   0   10  4   1   3   2   0   2   0   0    |  394  j     = sci.space
+0   0   0   0   0   3   1   0   27  2   291 0   11  25  0   0   1   0   13  18  0    |  392  k     = comp.sys.ibm.pc.hardware
+8   0   1   109 0   6   11  4   1   18  0   98  1   3   11  10  27  1   1   0   0    |  310  l     = talk.politics.misc
+0   11  0   0   0   3   6   0   10  6   11  0   299 13  0   2   13  0   7   8   0    |  389  m     = comp.graphics
+6   0   1   0   0   4   2   0   5   2   12  0   8   321 0   4   14  0   8   6   0    |  393  n     = sci.electronics
+2   0   0   0   0   0   4   1   0   3   1   0   3   1   372 6   0   2   1   2   0    |  398  o     = soc.religion.christian
+4   0   0   1   0   2   3   3   0   4   2   0   7   12  6   342 1   0   9   0   0    |  396  p     = sci.med
+0   1   0   1   0   1   4   0   3   0   1   0   8   4   0   2   369 0   1   1   0    |  396  q     = sci.crypt
+10  0   4   10  1   5   6   2   2   6   2   0   2   1   86  15  14  152 0   1   0    |  319  r     = alt.atheism
+4   0   0   0   0   9   1   1   8   1   12  0   3   6   0   2   0   0   341 2   0    |  390  s     = misc.forsale
+8   5   0   0   0   1   6   0   8   5   50  0   40  2   1   0   9   0   3   256 0    |  394  t     = comp.os.ms-windows.misc
+0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0    |  0    u     = unknown
+</PRE>
+</DIV></DIV>
+
+<H2><A name="TwentyNewsgroups-ComplementaryNaiveBayes"></A>Complementary Naive Bayes</H2>
+
+<P>To Train a CBayes Classifier using bi-grams</P>
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$&gt; $MAHOUT_HOME/bin/mahout trainclassifier \
+  -i 20news-input \
+  -o newsmodel \
+  -type cbayes \
+  -ng 2 \
+  -source hdfs
+</PRE>
+</DIV></DIV>
+
+<P>To Test a CBayes Classifier using bi-grams</P>
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$&gt; $MAHOUT_HOME/bin/mahout testclassifier \
+  -m newsmodel \
+  -d 20news-input \
+  -type cbayes \
+  -ng 2 \
+  -source hdfs \
+  -method mapreduce
+</PRE>
+</DIV></DIV>
+        </DIV>
+
+        
+      </DIV>
+    </DIV>
+    <DIV class="footer">
+      Generated by
+      <A href="http://www.atlassian.com/confluence/">Atlassian Confluence</A> (Version: 3.4.9 Build: 2042 Feb 14, 2011)
+      <A href="http://could.it/autoexport/">Auto Export Plugin</A> (Version: 1.0.0-dkulp)
+    </DIV>
+<SCRIPT type="text/javascript">
+
+  var _gaq = _gaq || [];
+  _gaq.push(['_setAccount', 'UA-17359171-1']);
+  _gaq.push(['_setDomainName', 'none']);
+  _gaq.push(['_setAllowLinker', true]);
+  _gaq.push(['_trackPageview']);
+
+  (function() {
+    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+  })();
+
+</SCRIPT>
+  </BODY>
+</HTML>
\ No newline at end of file

Added: mahout/site/new_website/MAHOUT/twentynewsgroups.html
URL: http://svn.apache.org/viewvc/mahout/site/new_website/MAHOUT/twentynewsgroups.html?rev=1243022&view=auto
==============================================================================
--- mahout/site/new_website/MAHOUT/twentynewsgroups.html (added)
+++ mahout/site/new_website/MAHOUT/twentynewsgroups.html Sat Feb 11 10:22:15 2012
@@ -0,0 +1,372 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<HTML>
+  <HEAD>
+    <LINK type="text/css" rel="stylesheet" href="resources/space.css">
+    <STYLE type="text/css">
+      .footer {
+        background-image:      url('http://cwiki.apache.org/confluence/images/border/border_bottom.gif');
+        background-repeat:     repeat-x;
+        background-position:   left top;
+        padding-top:           4px;
+        color:                 #666;
+      }
+    </STYLE>
+    <SCRIPT type="text/javascript" language="javascript">
+      var hide = null;
+      var show = null;
+      var children = null;
+
+      function init() {
+        /* Search form initialization */
+        var form = document.forms['search'];
+        if (form != null) {
+          form.elements['domains'].value = location.hostname;
+          form.elements['sitesearch'].value = location.hostname;
+        }
+
+        /* Children initialization */
+        hide = document.getElementById('hide');
+        show = document.getElementById('show');
+        children = document.all != null ?
+                   document.all['children'] :
+                   document.getElementById('children');
+        if (children != null) {
+          children.style.display = 'none';
+          show.style.display = 'inline';
+          hide.style.display = 'none';
+        }
+      }
+
+      function showChildren() {
+        children.style.display = 'block';
+        show.style.display = 'none';
+        hide.style.display = 'inline';
+      }
+
+      function hideChildren() {
+        children.style.display = 'none';
+        show.style.display = 'inline';
+        hide.style.display = 'none';
+      }
+    </SCRIPT>
+    <TITLE>TwentyNewsgroups</TITLE>
+  <META http-equiv="Content-Type" content="text/html;charset=UTF-8"></HEAD>
+  <BODY onload="init()">
+    <TABLE border="0" cellpadding="2" cellspacing="0" width="100%">
+      <TR class="topBar">
+        <TD align="left" valign="middle" class="topBarDiv" align="left" nowrap="">
+          &nbsp;<A href="index.html" title="Apache Lucene Mahout">Apache Lucene Mahout</A>&nbsp;&gt;&nbsp;<A href="index.html" title="index">index</A>&nbsp;&gt;&nbsp;<A href="quickstart.html" title="QuickStart">QuickStart</A>&nbsp;&gt;&nbsp;<A href="" title="TwentyNewsgroups">TwentyNewsgroups</A>
+        </TD>
+        <TD align="right" valign="middle" nowrap="">
+          <FORM name="search" action="http://www.google.com/search" method="get">
+            <INPUT type="hidden" name="ie" value="UTF-8">
+            <INPUT type="hidden" name="oe" value="UTF-8">
+            <INPUT type="hidden" name="domains" value="">
+            <INPUT type="hidden" name="sitesearch" value="">
+            <INPUT type="text" name="q" maxlength="255" value="">        
+            <INPUT type="submit" name="btnG" value="Google Search">
+          </FORM>
+        </TD>
+      </TR> 
+    </TABLE>
+
+    <DIV id="PageContent">
+      <DIV class="pageheader" style="padding: 6px 0px 0px 0px;">
+        <!-- We'll enable this once we figure out how to access (and save) the logo resource -->
+        <!--img src="/wiki/images/confluence_logo.gif" style="float: left; margin: 4px 4px 4px 10px;" border="0"-->
+        <DIV style="margin: 0px 10px 0px 10px" class="smalltext">Apache Lucene Mahout</DIV>
+        <DIV style="margin: 0px 10px 8px 10px" class="pagetitle">TwentyNewsgroups</DIV>
+
+        <DIV class="greynavbar" align="right" style="padding: 2px 10px; margin: 0px;">
+          <A href="http://cwiki.apache.org/confluence/pages/editpage.action?pageId=99739">
+            <IMG src="http://cwiki.apache.org/confluence/images/icons/notep_16.gif" height="16" width="16" border="0" align="absmiddle" title="Edit Page"></A>
+            <A href="http://cwiki.apache.org/confluence/pages/editpage.action?pageId=99739">Edit Page</A>
+          &nbsp;
+          <A href="http://cwiki.apache.org/confluence/pages/listpages.action?key=MAHOUT">
+            <IMG src="http://cwiki.apache.org/confluence/images/icons/browse_space.gif" height="16" width="16" border="0" align="absmiddle" title="Browse Space"></A>
+            <A href="http://cwiki.apache.org/confluence/pages/listpages.action?key=MAHOUT">Browse Space</A>
+          &nbsp;
+          <A href="http://cwiki.apache.org/confluence/pages/createpage.action?spaceKey=MAHOUT&fromPageId=99739">
+            <IMG src="http://cwiki.apache.org/confluence/images/icons/add_page_16.gif" height="16" width="16" border="0" align="absmiddle" title="Add Page"></A>
+          <A href="http://cwiki.apache.org/confluence/pages/createpage.action?spaceKey=MAHOUT&fromPageId=99739">Add Page</A>
+          &nbsp;
+          <A href="http://cwiki.apache.org/confluence/pages/createblogpost.action?spaceKey=MAHOUT&fromPageId=99739">
+            <IMG src="http://cwiki.apache.org/confluence/images/icons/add_blogentry_16.gif" height="16" width="16" border="0" align="absmiddle" title="Add News"></A>
+          <A href="http://cwiki.apache.org/confluence/pages/createblogpost.action?spaceKey=MAHOUT&fromPageId=99739">Add News</A>
+        </DIV>
+      </DIV>
+      <DIV class="pagesubheading" style="margin: 0px 10px 0px 10px;">
+        #editReport()
+      </DIV>
+
+      <DIV class="pagecontent">
+        <DIV class="wiki-content">
+          <H1><A name="TwentyNewsgroups-TwentyNewsgroupsClassification"></A>Twenty Newsgroups Classification</H1>
+
+<H2><A name="TwentyNewsgroups-Prerequisites"></A>Prerequisites</H2>
+
+<UL>
+	<LI>Mahout has been downloaded (<A href="index.html#index-Installation%252FSetup" class="external-link" rel="nofollow">instructions here</A>)</LI>
+	<LI>Maven is available</LI>
+	<LI>Your environment has the following variables:
+<DIV class="table-wrap">
+<TABLE class="confluenceTable"><TBODY>
+<TR>
+<TD class="confluenceTd"> <TT>HADOOP_HOME</TT> </TD>
+<TD class="confluenceTd"> Environment variables refers to where Hadoop lives </TD>
+</TR>
+<TR>
+<TD class="confluenceTd"> <TT>MAHOUT_HOME</TT> </TD>
+<TD class="confluenceTd"> Environment variables refers to where Mahout lives </TD>
+</TR>
+</TBODY></TABLE>
+</DIV>
+</LI>
+</UL>
+
+
+<H2><A name="TwentyNewsgroups-SettingupMahout"></A>Setting up Mahout</H2>
+
+<P>After downloading the distribution, unzip/untar it into the directory of your choice and do:</P>
+
+<OL>
+	<LI>In the trunk directory, compile everything and create the Hadoop job:
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>mvn install</PRE>
+</DIV></DIV></LI>
+</OL>
+
+
+<H3><A name="TwentyNewsgroups-ForMahout0.1%3A"></A>For Mahout 0.1:</H3>
+
+<P><!--  # If you've run this before, you may want to rm - rf the work and temp directories  --></P>
+
+<P>In <TT>$MAHOOT_HOME/examples</TT>:</P>
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ cd examples
+$ ant -f build-deprecated.xml get-files
+$ mkdir lib
+$ mvn dependency:copy-dependencies -DoutputDirectory=lib
+$ ant -f build-deprecated.xml extract-20news-18828 -Ddest=target
+$ mv 20news-18828-collapse 20news-input
+</PRE>
+</DIV></DIV>
+
+<P>NOTE: After you have done this, skip to the hadoop section to run the 20newsgroups example in mahout 0.1 release</P>
+
+<H3><A name="TwentyNewsgroups-ForMahout0.2%3A"></A>For Mahout 0.2+:</H3>
+
+<OL>
+	<LI>Change into <TT>$MAHOOT_HOME/examples</TT> directory
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ cd examples</PRE>
+</DIV></DIV></LI>
+	<LI>Download <TT>20news-18828.tar.gz</TT> from the <A href="http://people.csail.mit.edu/jrennie/20Newsgroups/" class="external-link" rel="nofollow">20newsgroups dataset</A></LI>
+	<LI>Extract dataset:
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ tar zxf 20news-18828.tar.gz</PRE>
+</DIV></DIV></LI>
+	<LI>Generate input dataset:
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ mkdir 20news-input
+$ mvn -e exec:java \
+      -Dexec.mainClass=org.apache.mahout.classifier.bayes.PrepareTwentyNewsgroups \
+      -Dexec.args=&quot;-p 20news-18828 \
+                   -o 20news-input \
+                   -a org.apache.lucene.analysis.standard.StandardAnalyzer \
+                   -c UTF-8&quot;
+</PRE>
+</DIV></DIV></LI>
+</OL>
+
+
+<H2><A name="TwentyNewsgroups-Runningtheexamplewithouthadoop"></A>Running the example without hadoop</H2>
+
+<P>This assumes you're running Mahout 0.2+:</P>
+
+<OL>
+	<LI>Train the classifier:
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ mvn -e exec:java \
+      -Dexec.mainClass=org.apache.mahout.classifier.bayes.TrainClassifier \
+      -Dexec.args=&quot;-i 20news-input \
+                   -o 20news-model \
+                   -type cbayes \
+                   -ng 1 \
+                   -source hdfs&quot;
+</PRE>
+</DIV></DIV></LI>
+	<LI>Test over input:
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ mvn -e exec:java \
+      -Dexec.mainClass=org.apache.mahout.classifier.bayes.TestClassifier \
+      -Dexec.args=&quot;-m 20news-model \
+                   -d 20news-input \
+                   -type cbayes \
+                   -ng 1 \
+                   -source hdfs \
+                   -method sequential&quot;
+</PRE>
+</DIV></DIV></LI>
+</OL>
+
+
+<H2><A name="TwentyNewsgroups-Running20newsgroupsexampleoverhadoopcluster"></A>Running 20newsgroups example over hadoop cluster</H2>
+
+<H3><A name="TwentyNewsgroups-SetUpHadoopCluster"></A>Set Up Hadoop Cluster</H3>
+<OL>
+	<LI>Edit <TT>hadoop-site.xml</TT>; add in local settings per <A href="http://hadoop.apache.org/core/docs/current/quickstart.html" class="external-link" rel="nofollow">Hadoop quickstart</A>
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>emacs $HADOOP_HOME/conf/hadoop-site.xml
+</PRE>
+</DIV></DIV></LI>
+	<LI>Format the HDFS
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ $HADOOP_HOME/bin/hadoop namenode -format
+</PRE>
+</DIV></DIV></LI>
+	<LI>Start Hadoop
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ $HADOOP_HOME/bin/start-all.sh
+</PRE>
+</DIV></DIV></LI>
+	<LI>Copy extracted text to HDFS
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$ $HADOOP_HOME/bin/hadoop dfs -put $MAHOUT_HOME/examples/20news-input 20news-input
+</PRE>
+</DIV></DIV></LI>
+</OL>
+
+
+<H3><A name="TwentyNewsgroups-TrainBayesClassifierUsingTrigrams"></A>Train Bayes Classifier Using Tri-grams</H3>
+
+<P>The following will run 4 map reduce jobs on Hadoop to train the classifier and will take a while on a single node machine.</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$HADOOP_HOME/bin/hadoop \
+    jar \
+    $MAHOUT_HOME/examples/target/mahout-examples-0.1.job \
+    org.apache.mahout.classifier.bayes.TrainClassifier \
+    -i 20news-input \
+    -o newsmodel \
+    -ng 3 \
+    -type bayes
+</PRE>
+</DIV></DIV>
+
+<P>You can monitor the status of these jobs by opening a web browser on your Job Tracker node: <A href="http://localhost:50030/jobtracker.jsp" class="external-link" rel="nofollow">http://localhost:50030/jobtracker.jsp</A></P>
+
+<P>Test classifier over the input folder</P>
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$HADOOP_HOME/bin/hadoop \
+    jar \
+    $MAHOUT_HOME/examples/target/mahout-examples-0.1.job \
+    org.apache.mahout.classifier.bayes.TestClassifier \
+    -p newsmodel \
+    -t work/20news-input \
+    -ng 3 \
+    -type bayes
+</PRE>
+</DIV></DIV>
+
+<P>Output might look like:</P>
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>08/11/07 16:52:39 INFO bayes.TestClassifier: Done loading model: # labels: 20
+08/11/07 16:52:39 INFO bayes.TestClassifier: Done generating Model
+08/11/07 16:52:57 INFO bayes.TestClassifier: alt.atheism96.9962453066333775/799.0
+08/11/07 16:53:15 INFO bayes.TestClassifier: comp.graphics99.28057553956835966/973.0
+08/11/07 16:53:45 INFO bayes.TestClassifier: comp.os.ms-windows.misc96.95431472081218955/985.0
+08/11/07 16:53:59 INFO bayes.TestClassifier: comp.sys.ibm.pc.hardware99.59266802443992978/982.0
+08/11/07 16:54:10 INFO bayes.TestClassifier: comp.sys.mac.hardware99.47970863683663956/961.0
+08/11/07 16:54:28 INFO bayes.TestClassifier: comp.windows.x99.59183673469387976/980.0
+08/11/07 16:54:38 INFO bayes.TestClassifier: misc.forsale98.45679012345678957/972.0
+08/11/07 16:54:50 INFO bayes.TestClassifier: rec.autos99.4949494949495985/990.0
+08/11/07 16:55:04 INFO bayes.TestClassifier: rec.motorcycles100.0994/994.0
+08/11/07 16:55:16 INFO bayes.TestClassifier: rec.sport.baseball99.89939637826961993/994.0
+08/11/07 16:55:36 INFO bayes.TestClassifier: rec.sport.hockey99.89989989989989998/999.0
+08/11/07 16:55:54 INFO bayes.TestClassifier: sci.crypt99.39455095862765985/991.0
+08/11/07 16:56:05 INFO bayes.TestClassifier: sci.electronics98.98063200815494971/981.0
+08/11/07 16:56:27 INFO bayes.TestClassifier: sci.med99.79797979797979988/990.0
+08/11/07 16:56:44 INFO bayes.TestClassifier: sci.space99.3920972644377981/987.0
+08/11/07 16:57:06 INFO bayes.TestClassifier: soc.religion.christian99.49849548645938992/997.0
+08/11/07 16:57:24 INFO bayes.TestClassifier: talk.politics.guns99.45054945054945905/910.0
+08/11/07 16:57:51 INFO bayes.TestClassifier: talk.politics.mideast98.82978723404256929/940.0
+08/11/07 16:58:13 INFO bayes.TestClassifier: talk.politics.misc89.93548387096774697/775.0
+08/11/07 16:58:25 INFO bayes.TestClassifier: talk.religion.misc61.78343949044586388/628.0
+08/11/07 16:58:25 INFO bayes.TestClassifier: =======================================================
+Summary
+-------------------------------------------------------
+Correctly Classified Instances          :      18369   97.5621%
+Incorrectly Classified Instances        :        459    2.4379%
+Total Classified Instances              :      18828
+
+=======================================================
+Confusion Matrix
+-------------------------------------------------------
+a    b    c    d    e    f    g    h    i    j    k    l    m    n    o    p    q    r    s    t    &lt;--Classified as
+994  0    0    0    0    0    0    0    0    0    0    0    0    0    0    0    0    0    0    0     |  994   a     = rec.motorcycles
+0    976  0    0    0    0    0    0    0    0    1    0    0    0    0    0    0    0    2    1     |  980   b     = comp.windows.x
+7    0    929  1    0    0    0    0    0    0    0    0    1    0    2    0    0    0    0    0     |  940   c     = talk.politics.mideast
+0    0    0    905  0    0    1    0    0    0    0    0    0    0    0    0    3    0    1    0     |  910   d     = talk.politics.guns
+4    1    4    27   388  1    0    1    0    5    1    1    2    2    149  7    2    33   0    0     |  628   e     = talk.religion.misc
+3    0    0    0    0    985  0    1    0    0    0    0    0    1    0    0    0    0    0    0     |  990   f     = rec.autos
+0    0    0    0    0    0    993  1    0    0    0    0    0    0    0    0    0    0    0    0     |  994   g     = rec.sport.baseball
+0    0    0    0    0    0    1    998  0    0    0    0    0    0    0    0    0    0    0    0     |  999   h     = rec.sport.hockey
+0    0    0    0    0    0    0    0    956  0    2    0    0    0    0    0    0    0    2    1     |  961   i     = comp.sys.mac.hardware
+0    0    0    0    0    0    0    0    0    981  0    0    5    0    0    1    0    0    0    0     |  987   j     = sci.space
+0    0    0    0    0    0    0    0    0    0    978  0    1    0    0    0    0    0    2    1     |  982   k     = comp.sys.ibm.pc.hardware
+1    0    3    36   0    1    2    1    0    5    0    697  4    0    3    3    19   0    0    0     |  775   l     = talk.politics.misc
+0    2    0    0    0    0    0    0    0    0    2    0    966  0    0    0    0    0    2    1     |  973   m     = comp.graphics
+1    0    0    0    0    0    0    0    0    0    6    0    0    971  0    0    0    0    3    0     |  981   n     = sci.electronics
+1    0    0    0    0    0    0    0    1    0    0    0    0    0    992  1    0    1    0    1     |  997   o     = soc.religion.christian
+0    0    0    0    0    0    0    0    0    0    1    0    0    0    0    988  0    0    0    1     |  990   p     = sci.med
+0    0    0    2    0    0    0    0    0    0    0    0    2    1    0    0    985  0    1    0     |  991   q     = sci.crypt
+0    0    0    1    1    0    0    0    0    1    0    0    1    0    19   0    1    775  0    0     |  799   r     = alt.atheism
+1    0    0    0    0    3    1    2    0    0    3    0    0    5    0    0    0    0    957  0     |  972   s     = misc.forsale
+0    0    0    8    0    0    0    0    0    0    6    0    6    0    0    0    0    0    10   955   |  985   t     = comp.os.ms-windows.misc
+</PRE>
+</DIV></DIV>
+
+<H2><A name="TwentyNewsgroups-ComplementaryNaiveBayes"></A>Complementary Naive Bayes</H2>
+
+<P>To Train a CBayes Classifier using bi-grams</P>
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$HADOOP_HOME/bin/hadoop \
+    jar \
+    $MAHOUT_HOME/examples/target/mahout-examples-0.1.job \
+    org.apache.mahout.classifier.bayes.TrainClassifier \
+    -i 20news-input \
+    -o 20news-model \
+    -ng 2 \
+    -type cbayes \
+    -source &lt;hdfs|hbase&gt;
+</PRE>
+</DIV></DIV>
+
+<P>To Test a CBayes Classifier using bi-grams</P>
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>$HADOOP_HOME/bin/hadoop \
+    jar \
+    $MAHOUT_HOME/examples/target/mahout-examples-0.1.job \
+    org.apache.mahout.classifier.bayes.TestClassifier \
+    -p 20news-model \
+    -t work/20news-input \
+    -ng 2 \
+    -type cbayes \
+    -source &lt;hdfs|hbase&gt; \
+    -method &lt;sequential|mapreduce&gt;
+</PRE>
+</DIV></DIV>
+        </DIV>
+
+        
+      </DIV>
+    </DIV>
+    <DIV class="footer">
+      Generated by
+      <A href="http://www.atlassian.com/confluence/">Atlassian Confluence</A> (Version: 3.2 Build: 1810 Mar 16, 2010)
+      <A href="http://could.it/autoexport/">Auto Export Plugin</A> (Version: 1.0.0-dkulp)
+    </DIV>
+  </BODY>
+</HTML>
\ No newline at end of file

Added: mahout/site/new_website/MAHOUT/use-an-existing-hadoop-ami.html
URL: http://svn.apache.org/viewvc/mahout/site/new_website/MAHOUT/use-an-existing-hadoop-ami.html?rev=1243022&view=auto
==============================================================================
--- mahout/site/new_website/MAHOUT/use-an-existing-hadoop-ami.html (added)
+++ mahout/site/new_website/MAHOUT/use-an-existing-hadoop-ami.html Sat Feb 11 10:22:15 2012
@@ -0,0 +1,372 @@
+
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<HTML>
+  <HEAD>
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/space.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/master.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/wiki-content.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/abs.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/menu.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/menu-ie.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/tables.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/panels.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/master-ie.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/renderer-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/content-types.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/login.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/information-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/layout-macros.css">
+<LINK type="text/css" rel="stylesheet" href="https://cwiki.apache.org/confluence/display/MAHOUT/$stylebase/default-theme.css">
+    <LINK type="text/css" rel="stylesheet" href="resources/space.css">
+    <STYLE type="text/css">
+      .footer {
+        background-image:      url('https://cwiki.apache.org/confluence/images/border/border_bottom.gif');
+        background-repeat:     repeat-x;
+        background-position:   left top;
+        padding-top:           4px;
+        color:                 #666;
+      }
+    </STYLE>
+    <SCRIPT type="text/javascript" language="javascript">
+      var hide = null;
+      var show = null;
+      var children = null;
+
+      function init() {
+        /* Search form initialization */
+        var form = document.forms['search'];
+        if (form != null) {
+          form.elements['domains'].value = location.hostname;
+          form.elements['sitesearch'].value = location.hostname;
+        }
+
+        /* Children initialization */
+        hide = document.getElementById('hide');
+        show = document.getElementById('show');
+        children = document.all != null ?
+                   document.all['children'] :
+                   document.getElementById('children');
+        if (children != null) {
+          children.style.display = 'none';
+          show.style.display = 'inline';
+          hide.style.display = 'none';
+        }
+      }
+
+      function showChildren() {
+        children.style.display = 'block';
+        show.style.display = 'none';
+        hide.style.display = 'inline';
+      }
+
+      function hideChildren() {
+        children.style.display = 'none';
+        show.style.display = 'inline';
+        hide.style.display = 'none';
+      }
+    </SCRIPT>
+    <TITLE>Use an Existing Hadoop AMI</TITLE>
+  <META http-equiv="Content-Type" content="text/html;charset=UTF-8"></HEAD>
+  <BODY onload="init()">
+    <TABLE border="0" cellpadding="2" cellspacing="0" width="100%">
+      <TR class="topBar">
+        <TD align="left" valign="middle" class="topBarDiv" align="left" nowrap="">
+          &nbsp;<A href="mahout-wiki.html" title="Apache Mahout">Apache Mahout</A>&nbsp;&gt;&nbsp;<A href="mahout-wiki.html" title="Mahout Wiki">Mahout Wiki</A>&nbsp;&gt;&nbsp;<A href="mahout-on-amazon-ec2.html" title="Mahout on Amazon EC2">Mahout on Amazon EC2</A>&nbsp;&gt;&nbsp;<A href="" title="Use an Existing Hadoop AMI">Use an Existing Hadoop AMI</A>
+        </TD>
+        <TD align="right" valign="middle" nowrap="">
+          <FORM name="search" action="http://www.google.com/search" method="get">
+            <INPUT type="hidden" name="ie" value="UTF-8">
+            <INPUT type="hidden" name="oe" value="UTF-8">
+            <INPUT type="hidden" name="domains" value="">
+            <INPUT type="hidden" name="sitesearch" value="">
+            <INPUT type="text" name="q" maxlength="255" value="">        
+            <INPUT type="submit" name="btnG" value="Google Search">
+          </FORM>
+        </TD>
+      </TR> 
+    </TABLE>
+
+    <DIV id="PageContent">
+      <DIV class="pageheader" style="padding: 6px 0px 0px 0px;">
+        <!-- We'll enable this once we figure out how to access (and save) the logo resource -->
+        <!--img src="/wiki/images/confluence_logo.gif" style="float: left; margin: 4px 4px 4px 10px;" border="0"-->
+        <DIV style="margin: 0px 10px 0px 10px" class="smalltext">Apache Mahout</DIV>
+        <DIV style="margin: 0px 10px 8px 10px" class="pagetitle">Use an Existing Hadoop AMI</DIV>
+
+        <DIV class="greynavbar" align="right" style="padding: 2px 10px; margin: 0px;">
+          <A href="https://cwiki.apache.org/confluence/pages/editpage.action?pageId=25204444">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/notep_16.gif" height="16" width="16" border="0" align="absmiddle" title="Edit Page"></A>
+            <A href="https://cwiki.apache.org/confluence/pages/editpage.action?pageId=25204444">Edit Page</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/listpages.action?key=MAHOUT">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/browse_space.gif" height="16" width="16" border="0" align="absmiddle" title="Browse Space"></A>
+            <A href="https://cwiki.apache.org/confluence/pages/listpages.action?key=MAHOUT">Browse Space</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/createpage.action?spaceKey=MAHOUT&fromPageId=25204444">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/add_page_16.gif" height="16" width="16" border="0" align="absmiddle" title="Add Page"></A>
+          <A href="https://cwiki.apache.org/confluence/pages/createpage.action?spaceKey=MAHOUT&fromPageId=25204444">Add Page</A>
+          &nbsp;
+          <A href="https://cwiki.apache.org/confluence/pages/createblogpost.action?spaceKey=MAHOUT&fromPageId=25204444">
+            <IMG src="https://cwiki.apache.org/confluence/images/icons/add_blogentry_16.gif" height="16" width="16" border="0" align="absmiddle" title="Add News"></A>
+          <A href="https://cwiki.apache.org/confluence/pages/createblogpost.action?spaceKey=MAHOUT&fromPageId=25204444">Add News</A>
+        </DIV>
+      </DIV>
+
+      <DIV class="pagecontent">
+        <DIV class="wiki-content">
+          <P>The following process was developed for launching Hadoop clusters in EC2 in order to benchmark Mahout's clustering algorithms using a large document set (see Mahout-588). Specifically, we used the ASF mail archives that have been parsed and converted to the Hadoop SequenceFile format (block-compressed) and saved to a public S3 folder: s3://asf-mail-archives/mahout-0.4/sequence-files. Overall, there are 6,094,444 key-value pairs in 283 files taking around 5.7GB of disk.</P>
+
+<P>You can also use Amazon's Elastic MapReduce, see <A href="mahout-on-elastic-mapreduce.html" class="external-link" rel="nofollow">Mahout on Elastic MapReduce</A>. However, using EC2 directly is slightly less expensive and provides greater visibility into the state of running jobs via the JobTracker Web UI. You can launch the EC2 cluster from your development machine; the following instructions were generated on Ubuntu workstation. We assume that you have successfully completed the Amazon EC2 Getting Started Guide, see <A href="http://docs.amazonwebservices.com/AWSEC2/latest/GettingStartedGuide/" class="external-link" rel="nofollow">EC2 Getting Started Guide</A>.</P>
+
+<P>Note, this work was supported in part by the Amazon Web Services Apache Projects Testing Program.</P>
+
+<H2><A name="UseanExistingHadoopAMI-LaunchHadoopCluster"></A>Launch Hadoop Cluster</H2>
+
+<H4><A name="UseanExistingHadoopAMI-GatherAmazonEC2keys%2Fsecuritycredentials"></A>Gather Amazon EC2 keys / security credentials</H4>
+
+<P>You will need the following:<BR>
+AWS Account ID<BR>
+Access Key ID<BR>
+Secret Access Key<BR>
+X.509 certificate and private key (e.g. cert-aws.pem and pk-aws.pem)<BR>
+EC2 Key-Pair (ssh public and private keys) for the US-EAST region.</P>
+
+<P>Please make sure the file permissions are &quot;<DEL>rw</DEL>------&quot; (e.g. chmod 600 gsg-keypair.pem). You can create a key-pair for the US-East region using the Amazon console. If you are confused about any of these terms, please see: <A href="http://alestic.com/2009/11/ec2-credentials" class="external-link" rel="nofollow">Understanding Access Credentials for AWS/EC2</A>.</P>
+
+<P>You should also export the EC2_PRIVATE_KEY and EC2_CERT environment variables to point to your AWS Certificate and Private Key files, for example:</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>export EC2_PRIVATE_KEY=$DEV/aws/pk-aws.pem
+export EC2_CERT=$DEV/aws/cert-aws.pem
+</PRE>
+</DIV></DIV>
+
+<P>These are used by the ec2-api-tools command to interact with Amazon Web Services.</P>
+
+<H4><A name="UseanExistingHadoopAMI-InstallandConfiguretheAmazonEC2APITools%3A"></A>Install and Configure the Amazon EC2 API Tools:</H4>
+
+<P>On Ubuntu, you'll need to enable the multi-verse in /etc/apt/sources.list to find the ec2-api-tools</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>apt-get update
+apt-get install ec2-api-tools
+</PRE>
+</DIV></DIV>
+
+<P>Once installed, verify you have access to EC2 by executing:</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>ec2-describe-images -x all | grep hadoop
+</PRE>
+</DIV></DIV>
+
+<H4><A name="UseanExistingHadoopAMI-InstallHadoop0.20.2Locally"></A>Install Hadoop 0.20.2 Locally</H4>
+
+<P>You need to install Hadoop locally in order to get access to the EC2 cluster deployment scripts. We use <TT>/mnt/dev</TT> as the base working directory because this process was originally conducted on an EC2 instance; be sure to replace this path with the correct path for your environment as you work through these steps.</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>sudo mkdir -p /mnt/dev/downloads
+sudo chown -R ubuntu:ubuntu /mnt/dev
+cd /mnt/dev/downloads
+wget http://apache.mirrors.hoobly.com//hadoop/core/hadoop-0.20.2/hadoop-0.20.2.tar.gz &amp;&amp; cd /mnt/dev &amp;&amp; tar zxvf downloads/hadoop-0.20.2.tar.gz
+ln -s hadoop-0.20.2 hadoop 
+</PRE>
+</DIV></DIV>
+
+<P>The scripts we need are in $HADOOP_HOME/scr/contrib/ec2. There are other approaches to deploying a Hadoop cluster on EC2, such as Cloudera's <A href="https://docs.cloudera.com/display/DOC/Cloudera%20Documentation%20Home%20Page" class="external-link" rel="nofollow">CDH3</A>. We chose to use the contrib/ec2 scripts because they are very easy to use provided there is an existing Hadoop AMI available.</P>
+
+<H4><A name="UseanExistingHadoopAMI-Edithadoopec2env.sh"></A>Edit hadoop-ec2-env.sh </H4>
+
+<P>Open hadoop/src/contrib/ec2/bin/hadoop-ec2-env.sh in your editor and set the Amazon security variables to match your environment, for example:</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>AWS_ACCOUNT_ID=####-####-####
+AWS_ACCESS_KEY_ID=???
+AWS_SECRET_ACCESS_KEY=???
+EC2_KEYDIR=/mnt/dev/aws
+KEY_NAME=gsg-keypair
+PRIVATE_KEY_PATH=/mnt/dev/aws/gsg-keypair.pem
+</PRE>
+</DIV></DIV>
+
+<P>The value of PRIVATE_KEY_PATH should be your EC2 key-pair pem file, such as /mnt/dev/aws/gsg-keypair.pem. This key-pair must be created in the US-East region.</P>
+
+<P>For Mahout, we recommended the following settings:</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>HADOOP_VERSION=0.20.2
+S3_BUCKET=453820947548/bixolabs-public-amis
+ENABLE_WEB_PORTS=true
+INSTANCE_TYPE=&quot;m1.xlarge&quot;
+</PRE>
+</DIV></DIV>
+
+<P>You do not need to worry about changing any variables below the comment that reads &quot;The following variables are only used when creating an AMI.&quot;.</P>
+
+<P>These settings will create a cluster of EC2 xlarge instances using the Hadoop 0.20.2 AMI provided by Bixo Labs.</P>
+
+<H4><A name="UseanExistingHadoopAMI-LaunchHadoopCluster"></A>Launch Hadoop Cluster</H4>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>cd $HADOOP_HOME/src/contrib/ec2
+bin/hadoop-ec2 launch-cluster mahout-clustering 2
+</PRE>
+</DIV></DIV>
+
+<P>This will launch 3 xlarge instances (two workers + one for the NameNode aka &quot;master&quot;). It may take up to 5 minutes to launch a cluster named &quot;mahout-clustering&quot;; watch the console for errors. The cluster will launch in the US-East region so you won't incur any data transfer fees to/from US-Standard S3 buckets. You can re-use the cluster name for launching other clusters of different sizes. Behind the scenes, the Hadoop scripts will create two EC2 security groups that configure the firewall for accessing your Hadoop cluster.</P>
+
+<H4><A name="UseanExistingHadoopAMI-Launchproxy"></A>Launch proxy</H4>
+
+<P>Assuming your cluster launched successfully, establish a SOCKS tunnel to your master node to access the JobTracker Web UI from your local browser.</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>bin/hadoop-ec2 proxy mahout-clustering &amp;
+</PRE>
+</DIV></DIV>
+
+<P>This command will output the URLs for the JobTracker and NameNode Web UI, such as:</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>JobTracker http://ec2-???-???-???-???.compute-1.amazonaws.com:50030
+</PRE>
+</DIV></DIV>
+
+<H4><A name="UseanExistingHadoopAMI-SetupFoxyProxy%28FireFoxplugin%29"></A>Setup FoxyProxy (FireFox plug-in)</H4>
+
+<P>Once the FoxyProxy plug-in is installed in FireFox, go to Options &gt; FoxyProxy Standard &gt; Options to setup a proxy on localhost:6666 for the JobTracker and NameNode Web UI URLs from the previous step. For more information about FoxyProxy, please see: <A href="http://getfoxyproxy.org/downloads.html" class="external-link" rel="nofollow">FoxyProxy</A></P>
+
+<P>Now you are ready to run Mahout jobs in your cluster.</P>
+
+<H2><A name="UseanExistingHadoopAMI-LaunchClusteringJobfromMasterserver"></A>Launch Clustering Job from Master server</H2>
+
+<H4><A name="UseanExistingHadoopAMI-Logintothemasterserver%3A"></A>Login to the master server:</H4>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>bin/hadoop-ec2 login mahout-clustering
+</PRE>
+</DIV></DIV>
+
+<P>Hadoop does not start until all EC2 instances are running, look for java processes on the master server using: ps waux | grep java</P>
+
+<H4><A name="UseanExistingHadoopAMI-InstallMahout"></A>Install Mahout</H4>
+
+<P>Since this is EC2, you have the most disk space on the master node in /mnt.</P>
+
+<H5><A name="UseanExistingHadoopAMI-Fromadistribution"></A>From a distribution</H5>
+
+<P>NOTE: Substitute in the appropriate version number/URLs as necessary.  0.4 is not the latest version of Mahout.</P>
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>mkdir -p /mnt/dev/downloads
+cd /mnt/dev/downloads
+wget http://apache.mesi.com.ar//mahout/0.4/mahout-distribution-0.4.tar.gz &amp;&amp; cd /mnt/dev &amp;&amp; tar zxvf downloads/mahout-distribution-0.4.tar.gz
+ln -s mahout-distribution-0.4 mahout
+</PRE>
+</DIV></DIV>
+
+<H5><A name="UseanExistingHadoopAMI-FromSource"></A>From Source</H5>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>Install Subversion: &gt;yum install subversion //Note, you can also use Git, so substitute in the appropriate URL
+&gt; svn co http://svn.apache.org/repos/asf/mahout/trunk mahout/trunk
+Install Maven 3.x and put it in the path
+&gt; cd mahout/trunk
+&gt; mvn install //Optionally add -DskipTests
+</PRE>
+</DIV></DIV>
+
+<H4><A name="UseanExistingHadoopAMI-ConfigureHadoop"></A>Configure Hadoop</H4>
+
+<P>You'll want to increase the Max Heap Size for the data nodes (mapred.child.java.opts) and set the correct number of reduce tasks based on the size of your cluster. </P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>vi $HADOOP_HOME/conf/hadoop-site.xml
+</PRE>
+</DIV></DIV>
+
+<P>(NOTE: if this file doesn't exist yet, then the cluster nodes are still starting up. Wait a few minutes and then try again.)</P>
+
+<P>Add the following properties:</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>&lt;!-- Change 6 to the correct number for your cluster --&gt;
+&lt;property&gt;
+  &lt;name&gt;mapred.reduce.tasks&lt;/name&gt;
+  &lt;value&gt;6&lt;/value&gt;
+&lt;/property&gt;
+
+&lt;property&gt;
+  &lt;name&gt;mapred.child.java.opts&lt;/name&gt;
+  &lt;value&gt;-Xmx4096m&lt;/value&gt;
+&lt;/property&gt;
+</PRE>
+</DIV></DIV>
+
+<P>You can safely run 3 reducers per node on EC2 xlarge instances with 4GB of max heap each. If you are using large instances, then you may be able to have 2 per node or only 1 if your jobs are CPU intensive.</P>
+
+<H4><A name="UseanExistingHadoopAMI-CopythevectorsfromS3toHDFS"></A>Copy the vectors from S3 to HDFS</H4>
+
+<P>Use Hadoop's distcp command to copy the vectors from S3 to HDFS.</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>hadoop distcp -Dmapred.task.timeout=1800000 \
+s3n://ACCESS_KEY:SECRET_KEY@asf-mail-archives/mahout-0.4/sparse-1-gram-stem/tfidf-vectors \
+/asf-mail-archives/mahout-0.4/tfidf-vectors
+</PRE>
+</DIV></DIV>
+
+<P>The files are stored in the US-Standard S3 bucket so there is no charge for data transfer to your EC2 cluster, as it is running in the US-EAST region.</P>
+
+<H4><A name="UseanExistingHadoopAMI-Launchtheclusteringjob%28fromthemasterserver%29"></A>Launch the clustering job (from the master server)</H4>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>cd /mnt/dev/mahout
+bin/mahout kmeans -i /asf-mail-archives/mahout-0.4/tfidf-vectors/ \
+  -c /asf-mail-archives/mahout-0.4/initial-clusters/ \
+  -o /asf-mail-archives/mahout-0.4/kmeans-clusters/ \
+  --numClusters 100 \
+  --maxIter 10 \
+  --distanceMeasure org.apache.mahout.common.distance.CosineDistanceMeasure \
+  --convergenceDelta 0.01 &amp;
+</PRE>
+</DIV></DIV>
+
+<P>You can monitor the job using the JobTracker Web UI through FoxyProxy.</P>
+
+<H4><A name="UseanExistingHadoopAMI-DumpClusters"></A>Dump Clusters</H4>
+
+<P>Once completed, you can view the results using Mahout's cluster dumper</P>
+
+<DIV class="preformatted panel" style="border-width: 1px;"><DIV class="preformattedContent panelContent">
+<PRE>bin/mahout clusterdump --seqFileDir /asf-mail-archives/mahout-0.4/kmeans-clusters/clusters-1/ \
+  --numWords 20 \
+  --dictionary s3n://ACCESS_KEY:SECRET_KEY@asf-mail-archives/mahout-0.4/sparse-1-gram-stem/dictionary.file-0 \
+  --dictionaryType sequencefile --output clusters.txt --substring 100
+</PRE>
+</DIV></DIV>
+        </DIV>
+
+        
+      </DIV>
+    </DIV>
+    <DIV class="footer">
+      Generated by
+      <A href="http://www.atlassian.com/confluence/">Atlassian Confluence</A> (Version: 3.4.9 Build: 2042 Feb 14, 2011)
+      <A href="http://could.it/autoexport/">Auto Export Plugin</A> (Version: 1.0.0-dkulp)
+    </DIV>
+<SCRIPT type="text/javascript">
+
+  var _gaq = _gaq || [];
+  _gaq.push(['_setAccount', 'UA-17359171-1']);
+  _gaq.push(['_setDomainName', 'none']);
+  _gaq.push(['_setAllowLinker', true]);
+  _gaq.push(['_trackPageview']);
+
+  (function() {
+    var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+    ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+    var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+  })();
+
+</SCRIPT>
+  </BODY>
+</HTML>
\ No newline at end of file