You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by to...@apache.org on 2012/10/12 13:57:27 UTC
svn commit: r1397525 - in /hama/trunk/examples/src:
main/java/org/apache/hama/examples/GradientDescentExample.java
test/java/org/apache/hama/examples/GradientDescentTest.java test/resources/
test/resources/gd_file_format.txt
Author: tommaso
Date: Fri Oct 12 11:57:26 2012
New Revision: 1397525
URL: http://svn.apache.org/viewvc?rev=1397525&view=rev
Log:
[HAMA-651] - adding gradient descent to examples
Added:
hama/trunk/examples/src/main/java/org/apache/hama/examples/GradientDescentExample.java (with props)
hama/trunk/examples/src/test/java/org/apache/hama/examples/GradientDescentTest.java (with props)
hama/trunk/examples/src/test/resources/
hama/trunk/examples/src/test/resources/gd_file_format.txt (with props)
Added: hama/trunk/examples/src/main/java/org/apache/hama/examples/GradientDescentExample.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/GradientDescentExample.java?rev=1397525&view=auto
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/GradientDescentExample.java (added)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/GradientDescentExample.java Fri Oct 12 11:57:26 2012
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.bsp.BSPJob;
+import org.apache.hama.bsp.FileOutputFormat;
+import org.apache.hama.bsp.TextOutputFormat;
+import org.apache.hama.ml.regression.GradientDescentBSP;
+import org.apache.hama.ml.regression.VectorDoubleFileInputFormat;
+import org.apache.hama.ml.writable.VectorWritable;
+
+/**
+ * A {@link GradientDescentBSP} job example
+ */
+public class GradientDescentExample {
+ private static final Path TMP_OUTPUT = new Path("/tmp/gd");
+
+ public static void main(String[] args) throws InterruptedException,
+ IOException, ClassNotFoundException {
+ // BSP job configuration
+ HamaConfiguration conf = new HamaConfiguration();
+
+ BSPJob bsp = new BSPJob(conf, GradientDescentExample.class);
+ // Set the job name
+ bsp.setJobName("Gradient Descent Example");
+ bsp.setBspClass(GradientDescentBSP.class);
+ bsp.setInputFormat(VectorDoubleFileInputFormat.class);
+ bsp.setInputPath(new Path(args[0]));
+ bsp.setInputKeyClass(VectorWritable.class);
+ bsp.setInputValueClass(DoubleWritable.class);
+ bsp.setOutputKeyClass(VectorWritable.class);
+ bsp.setOutputValueClass(DoubleWritable.class);
+ bsp.setOutputFormat(TextOutputFormat.class);
+ FileOutputFormat.setOutputPath(bsp, TMP_OUTPUT);
+ bsp.setNumBspTask(3);
+
+ long startTime = System.currentTimeMillis();
+ if (bsp.waitForCompletion(true)) {
+ printOutput(conf);
+ System.out.println("Job Finished in "
+ + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds");
+ }
+
+ }
+
+ static void printOutput(HamaConfiguration conf) throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ FileStatus[] files = fs.listStatus(TMP_OUTPUT);
+ for (int i = 0; i < files.length; i++) {
+ if (files[i].getLen() > 0) {
+ FSDataInputStream in = fs.open(files[i].getPath());
+ IOUtils.copyBytes(in, System.out, conf, false);
+ in.close();
+ break;
+ }
+ }
+
+ fs.delete(TMP_OUTPUT, true);
+ }
+}
Propchange: hama/trunk/examples/src/main/java/org/apache/hama/examples/GradientDescentExample.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: hama/trunk/examples/src/test/java/org/apache/hama/examples/GradientDescentTest.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/test/java/org/apache/hama/examples/GradientDescentTest.java?rev=1397525&view=auto
==============================================================================
--- hama/trunk/examples/src/test/java/org/apache/hama/examples/GradientDescentTest.java (added)
+++ hama/trunk/examples/src/test/java/org/apache/hama/examples/GradientDescentTest.java Fri Oct 12 11:57:26 2012
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples;
+
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Testcase for {@link GradientDescentExample}
+ */
+public class GradientDescentTest {
+ @Test
+ public void testCorrectGDExecution() throws Exception {
+ GradientDescentExample.main(new String[]{"src/test/resources/gd_file_format.txt"});
+ }
+
+ @Test
+ public void testWrongGDExecutionWithEmptyArgs() {
+ try {
+ GradientDescentExample.main(new String[0]);
+ fail("GradientDescentExample should fail if the argument list has size 0");
+ } catch (Exception e) {
+ // everything ok
+ }
+ }
+
+}
Propchange: hama/trunk/examples/src/test/java/org/apache/hama/examples/GradientDescentTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: hama/trunk/examples/src/test/resources/gd_file_format.txt
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/test/resources/gd_file_format.txt?rev=1397525&view=auto
==============================================================================
--- hama/trunk/examples/src/test/resources/gd_file_format.txt (added)
+++ hama/trunk/examples/src/test/resources/gd_file_format.txt Fri Oct 12 11:57:26 2012
@@ -0,0 +1,6 @@
+0.1>0.1 0.2 0.3 0.4 0.5 0.6 0.7
+0.12>0.3 0.4 0.5 0.6 0.9 0.1 0.3
+0.22>0.1 0.1 0.3 0.1 0.1 0.01 0.1
+0.4>0.2 0.4 0.1 0.1 0.4 0.1 0.8
+0.3>0.3 0.4 0.5 0.6 0.7 0.8 0.9
+0.71>0.1 0.3 0.4 0.1 0.4 0.5 0.1
Propchange: hama/trunk/examples/src/test/resources/gd_file_format.txt
------------------------------------------------------------------------------
svn:eol-style = native