You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2012/12/12 13:09:49 UTC
svn commit: r1420632 - in /hama/trunk: ./
examples/src/main/java/org/apache/hama/examples/
examples/src/main/java/org/apache/hama/examples/util/
examples/src/test/java/org/apache/hama/examples/
Author: edwardyoon
Date: Wed Dec 12 12:09:48 2012
New Revision: 1420632
URL: http://svn.apache.org/viewvc?rev=1420632&view=rev
Log:
Add SpMV example
Added:
hama/trunk/examples/src/main/java/org/apache/hama/examples/SpMV.java
hama/trunk/examples/src/main/java/org/apache/hama/examples/util/DenseVectorWritable.java
hama/trunk/examples/src/main/java/org/apache/hama/examples/util/SparseVectorWritable.java
hama/trunk/examples/src/test/java/org/apache/hama/examples/SpMVTest.java
Modified:
hama/trunk/CHANGES.txt
hama/trunk/examples/src/main/java/org/apache/hama/examples/util/Generator.java
Modified: hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hama/trunk/CHANGES.txt?rev=1420632&r1=1420631&r2=1420632&view=diff
==============================================================================
--- hama/trunk/CHANGES.txt (original)
+++ hama/trunk/CHANGES.txt Wed Dec 12 12:09:48 2012
@@ -4,6 +4,7 @@ Release 0.7 (unreleased changes)
NEW FEATURES
+ HAMA-524: Add SpMV example (Mikalai Parafeniuk via edwardyoon)
HAMA-658: Add random symmetric sparse matrix generator (edwardyoon)
BUG FIXES
Added: hama/trunk/examples/src/main/java/org/apache/hama/examples/SpMV.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/SpMV.java?rev=1420632&view=auto
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/SpMV.java (added)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/SpMV.java Wed Dec 12 12:09:48 2012
@@ -0,0 +1,330 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.bsp.BSP;
+import org.apache.hama.bsp.BSPJob;
+import org.apache.hama.bsp.BSPJobClient;
+import org.apache.hama.bsp.BSPPeer;
+import org.apache.hama.bsp.ClusterStatus;
+import org.apache.hama.bsp.FileOutputFormat;
+import org.apache.hama.bsp.SequenceFileInputFormat;
+import org.apache.hama.bsp.SequenceFileOutputFormat;
+import org.apache.hama.bsp.sync.SyncException;
+import org.apache.hama.examples.util.DenseVectorWritable;
+import org.apache.hama.examples.util.SparseVectorWritable;
+import org.apache.hama.util.KeyValuePair;
+
+/**
+ * Sparse matrix vector multiplication. Currently it uses row-wise access.
+ * Assumptions: 1) each peer should have copy of input vector for efficient
+ * operations. 2) row-wise implementation is good because we don't need to care
+ * about communication 3) the main way to improve performance - create custom
+ * Partitioner
+ *
+ * TODO need to be simplified.
+ */
+public class SpMV {
+
+ protected static final Log LOG = LogFactory.getLog(SpMV.class);
+ private static String resultPath;
+ private static final String outputPathString = "spmv.outputpath";
+ private static final String inputMatrixPathString = "spmv.inputmatrixpath";
+ private static final String inputVectorPathString = "spmv.inputvectorpath";
+ private static String requestedBspTasksString = "bsptask.count";
+ private static final String intermediate = "/part";
+
+ enum RowCounter {
+ TOTAL_ROWS
+ }
+
+ public static String getResultPath() {
+ return resultPath;
+ }
+
+ public static void setResultPath(String resultPath) {
+ SpMV.resultPath = resultPath;
+ }
+
+ /**
+ * IMPORTANT: This can be a bottle neck. Problem can be here{@core
+ * WritableUtil.convertSpMVOutputToDenseVector()}
+ */
+ private static void convertToDenseVector(Configuration conf)
+ throws IOException {
+ String resultPath = convertSpMVOutputToDenseVector(
+ conf.get(outputPathString), conf);
+ setResultPath(resultPath);
+ }
+
+ /**
+ * This class performs sparse matrix vector multiplication. u = m * v.
+ */
+ private static class SpMVBSP
+ extends
+ BSP<IntWritable, SparseVectorWritable, IntWritable, DoubleWritable, NullWritable> {
+ private DenseVectorWritable v;
+
+ /**
+ * Each peer reads input dense vector.
+ */
+ @Override
+ public void setup(
+ BSPPeer<IntWritable, SparseVectorWritable, IntWritable, DoubleWritable, NullWritable> peer)
+ throws IOException, SyncException, InterruptedException {
+ // reading input vector, which represented as matrix row
+ Configuration conf = peer.getConfiguration();
+ v = new DenseVectorWritable();
+ readFromFile(conf.get(inputVectorPathString), v, conf);
+ peer.sync();
+ }
+
+ /**
+ * Local inner product computation and output.
+ */
+ @Override
+ public void bsp(
+ BSPPeer<IntWritable, SparseVectorWritable, IntWritable, DoubleWritable, NullWritable> peer)
+ throws IOException, SyncException, InterruptedException {
+ KeyValuePair<IntWritable, SparseVectorWritable> row = null;
+ while ((row = peer.readNext()) != null) {
+ // it will be needed in conversion of output to result vector
+ peer.getCounter(RowCounter.TOTAL_ROWS).increment(1L);
+ int key = row.getKey().get();
+ double sum = 0;
+ SparseVectorWritable mRow = row.getValue();
+ if (v.getSize() != mRow.getSize())
+ throw new RuntimeException("Matrix row with index = " + key
+ + " is not consistent with input vector. Row size = "
+ + mRow.getSize() + " vector size = " + v.getSize());
+ List<Integer> mIndeces = mRow.getIndeces();
+ List<Double> mValues = mRow.getValues();
+ for (int i = 0; i < mIndeces.size(); i++)
+ sum += v.get(mIndeces.get(i)) * mValues.get(i);
+ peer.write(new IntWritable(key), new DoubleWritable(sum));
+ }
+ }
+
+ }
+
+ /**
+ * Method which actually starts SpMV.
+ */
+ private static void startTask(HamaConfiguration conf) throws IOException,
+ InterruptedException, ClassNotFoundException {
+ BSPJob bsp = new BSPJob(conf, SpMV.class);
+ bsp.setJobName("Sparse matrix vector multiplication");
+ bsp.setBspClass(SpMVBSP.class);
+ /*
+ * Input matrix is presented as pairs of integer and SparseVectorWritable.
+ * Output is pairs of integer and double
+ */
+ bsp.setInputFormat(SequenceFileInputFormat.class);
+ bsp.setOutputKeyClass(IntWritable.class);
+ bsp.setOutputValueClass(DoubleWritable.class);
+ bsp.setOutputFormat(SequenceFileOutputFormat.class);
+ bsp.setInputPath(new Path(conf.get(inputMatrixPathString)));
+
+ FileOutputFormat.setOutputPath(bsp, new Path(conf.get(outputPathString)));
+
+ BSPJobClient jobClient = new BSPJobClient(conf);
+ ClusterStatus cluster = jobClient.getClusterStatus(true);
+
+ int requestedTasks = conf.getInt(requestedBspTasksString, -1);
+ if (requestedTasks != -1) {
+ bsp.setNumBspTask(requestedTasks);
+ } else {
+ bsp.setNumBspTask(cluster.getMaxTasks());
+ }
+
+ long startTime = System.currentTimeMillis();
+ if (bsp.waitForCompletion(true)) {
+ LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime)
+ / 1000.0 + " seconds.");
+ convertToDenseVector(conf);
+ LOG.info("Result is in " + getResultPath());
+ } else {
+ setResultPath(null);
+ }
+ }
+
+ private static void printUsage() {
+ LOG.info("Usage: spmv <Matrix> <Vector> <output> [number of tasks (default max)]");
+ }
+
+ /**
+ * Function parses command line in standart form.
+ */
+ private static void parseArgs(HamaConfiguration conf, String[] args) {
+ if (args.length < 3) {
+ printUsage();
+ System.exit(-1);
+ }
+
+ conf.set(inputMatrixPathString, args[0]);
+ conf.set(inputVectorPathString, args[1]);
+
+ Path path = new Path(args[2]);
+ path = path.suffix(intermediate);
+ conf.set(outputPathString, path.toString());
+
+ if (args.length == 4) {
+ try {
+ int taskCount = Integer.parseInt(args[3]);
+ if (taskCount < 0) {
+ printUsage();
+ throw new IllegalArgumentException(
+ "The number of requested tasks can't be negative. Actual value: "
+ + String.valueOf(taskCount));
+ }
+ conf.setInt(requestedBspTasksString, taskCount);
+ } catch (NumberFormatException e) {
+ printUsage();
+ throw new IllegalArgumentException(
+ "The format of requested task count is int. Can not parse value: "
+ + args[3]);
+ }
+ }
+ }
+
+ /**
+ * SpMV produces a file, which contains result dense vector in format of pairs
+ * of integer and double. The aim of this method is to convert SpMV output to
+ * format usable in subsequent computation - dense vector. It can be usable
+ * for iterative solvers. IMPORTANT: currently it is used in SpMV. It can be a
+ * bottle neck, because all input needs to be stored in memory.
+ *
+ * @param SpMVoutputPathString output path, which represents directory with
+ * part files.
+ * @param conf configuration
+ * @return path to output vector.
+ * @throws IOException
+ */
+ public static String convertSpMVOutputToDenseVector(
+ String SpMVoutputPathString, Configuration conf) throws IOException {
+ List<Integer> indeces = new ArrayList<Integer>();
+ List<Double> values = new ArrayList<Double>();
+
+ FileSystem fs = FileSystem.get(conf);
+ Path SpMVOutputPath = new Path(SpMVoutputPathString);
+ Path resultOutputPath = SpMVOutputPath.getParent().suffix("/result");
+ FileStatus[] stats = fs.listStatus(SpMVOutputPath);
+ for (FileStatus stat : stats) {
+ String filePath = stat.getPath().toUri().getPath();
+ SequenceFile.Reader reader = null;
+ fs.open(new Path(filePath));
+ try {
+ reader = new SequenceFile.Reader(fs, new Path(filePath), conf);
+ IntWritable key = new IntWritable();
+ DoubleWritable value = new DoubleWritable();
+ while (reader.next(key, value)) {
+ indeces.add(key.get());
+ values.add(value.get());
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (reader != null)
+ reader.close();
+ }
+ }
+ DenseVectorWritable result = new DenseVectorWritable();
+ result.setSize(indeces.size());
+ for (int i = 0; i < indeces.size(); i++)
+ result.addCell(indeces.get(i), values.get(i));
+ writeToFile(resultOutputPath.toString(), result, conf);
+ return resultOutputPath.toString();
+ }
+
+ public static void readFromFile(String pathString, Writable result,
+ Configuration conf) throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ SequenceFile.Reader reader = null;
+ Path path = new Path(pathString);
+ List<String> filePaths = new ArrayList<String>();
+ if (!fs.isFile(path)) {
+ FileStatus[] stats = fs.listStatus(path);
+ for (FileStatus stat : stats) {
+ filePaths.add(stat.getPath().toUri().getPath());
+ }
+ } else if (fs.isFile(path)) {
+ filePaths.add(path.toString());
+ }
+ try {
+ for (String filePath : filePaths) {
+ reader = new SequenceFile.Reader(fs, new Path(filePath), conf);
+ IntWritable key = new IntWritable();
+ reader.next(key, result);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (reader != null)
+ reader.close();
+ }
+ }
+
+ /**
+ * This method is used to write vector from memory to specified path.
+ *
+ * @param pathString output path
+ * @param result instance of vector to be writed
+ * @param conf configuration
+ * @throws IOException
+ */
+ public static void writeToFile(String pathString, Writable result,
+ Configuration conf) throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ SequenceFile.Writer writer = null;
+ try {
+ writer = new SequenceFile.Writer(fs, conf, new Path(pathString),
+ IntWritable.class, result.getClass());
+ IntWritable key = new IntWritable();
+ writer.append(key, result);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (writer != null)
+ writer.close();
+ }
+ }
+
+ public static void main(String[] args) throws IOException,
+ InterruptedException, ClassNotFoundException {
+ HamaConfiguration conf = new HamaConfiguration();
+ parseArgs(conf, args);
+ startTask(conf);
+ }
+
+}
Added: hama/trunk/examples/src/main/java/org/apache/hama/examples/util/DenseVectorWritable.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/util/DenseVectorWritable.java?rev=1420632&view=auto
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/util/DenseVectorWritable.java (added)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/util/DenseVectorWritable.java Wed Dec 12 12:09:48 2012
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples.util;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class represents dense vector. It will improve memory consumption up to
+ * two times in comparison to SparseVectorWritable in case of vectors
+ * which sparsity is close to 1. Internally represents vector values as array.
+ * Can be used in SpMV for representation of input and output vector.
+ */
+public class DenseVectorWritable implements Writable {
+
+ private double values[];
+
+ public DenseVectorWritable() {
+ values = new double[0];
+ }
+
+ public int getSize() {
+ return values.length;
+ }
+
+ public void setSize(int size) {
+ values = new double[size];
+ }
+
+ public double get(int index) {
+ return values[index];
+ }
+
+ public void addCell(int index, double value) {
+ values[index] = value;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ int size = in.readInt();
+ int len = in.readInt();
+ setSize(size);
+ for (int i = 0; i < len; i++) {
+ int index = in.readInt();
+ double value = in.readDouble();
+ values[index] = value;
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(getSize());
+ out.writeInt(getSize());
+ for (int i = 0; i < getSize(); i++) {
+ out.writeInt(i);
+ out.writeDouble(values[i]);
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder st = new StringBuilder();
+ st.append(" "+getSize()+" "+getSize());
+ for (int i = 0; i < getSize(); i++)
+ st.append(" "+i+" "+values[i]);
+ return st.toString();
+ }
+
+}
Modified: hama/trunk/examples/src/main/java/org/apache/hama/examples/util/Generator.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/util/Generator.java?rev=1420632&r1=1420631&r2=1420632&view=diff
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/util/Generator.java (original)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/util/Generator.java Wed Dec 12 12:09:48 2012
@@ -24,13 +24,18 @@ public class Generator {
System.out.println("Valid command names are:");
System.out
.println(" symmetric: Generate random symmetric matrix, which can be used as a input of graph examples.");
+ System.out.println(" square: Generate random square matrix.");
System.exit(1);
}
+ String[] newArgs = new String[args.length - 1];
+ System.arraycopy(args, 1, newArgs, 0, args.length - 1);
+
if (args[0].equals("symmetric")) {
- String[] newArgs = new String[args.length - 1];
- System.arraycopy(args, 1, newArgs, 0, args.length - 1);
SymmetricMatrixGen.main(newArgs);
+ } else if(args[0].equals("square")) {
+ System.out.println("Not implemented yet.");
+ //SquareMatrixGen.main(newArgs);
}
}
}
Added: hama/trunk/examples/src/main/java/org/apache/hama/examples/util/SparseVectorWritable.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/main/java/org/apache/hama/examples/util/SparseVectorWritable.java?rev=1420632&view=auto
==============================================================================
--- hama/trunk/examples/src/main/java/org/apache/hama/examples/util/SparseVectorWritable.java (added)
+++ hama/trunk/examples/src/main/java/org/apache/hama/examples/util/SparseVectorWritable.java Wed Dec 12 12:09:48 2012
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples.util;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * This class represents sparse vector. It will give improvement in memory
+ * consumption in case of vectors which sparsity is close to zero. Can be used
+ * in SpMV for representing input matrix rows efficiently. Internally
+ * represents values as list of indeces and list of values.
+ */
+public class SparseVectorWritable implements Writable {
+
+ private Integer size;
+ private List<Integer> indeces;
+ private List<Double> values;
+
+ public SparseVectorWritable() {
+ indeces = new ArrayList<Integer>();
+ values = new ArrayList<Double>();
+ }
+
+ public void clear(){
+ indeces = new ArrayList<Integer>();
+ values = new ArrayList<Double>();
+ }
+
+ public void addCell(int index, double value) {
+ indeces.add(index);
+ values.add(value);
+ }
+
+ public void setSize(int size) {
+ this.size = size;
+ }
+
+ public int getSize() {
+ if (size != null)
+ return size;
+ return indeces.size();
+ }
+
+ public List<Integer> getIndeces() {
+ return indeces;
+ }
+
+ public List<Double> getValues() {
+ return values;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ clear();
+ int size = in.readInt();
+ int len = in.readInt();
+ setSize(size);
+ for (int i = 0; i < len; i++) {
+ int index = in.readInt();
+ double value = in.readDouble();
+ this.addCell(index, value);
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(getSize());
+ out.writeInt(indeces.size());
+ for (int i = 0; i < indeces.size(); i++) {
+ out.writeInt(indeces.get(i));
+ out.writeDouble(values.get(i));
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder st = new StringBuilder();
+ st.append(" "+getSize()+" "+indeces.size());
+ for (int i = 0; i < indeces.size(); i++)
+ st.append(" "+indeces.get(i)+" "+values.get(i));
+ return st.toString();
+ }
+
+}
\ No newline at end of file
Added: hama/trunk/examples/src/test/java/org/apache/hama/examples/SpMVTest.java
URL: http://svn.apache.org/viewvc/hama/trunk/examples/src/test/java/org/apache/hama/examples/SpMVTest.java?rev=1420632&view=auto
==============================================================================
--- hama/trunk/examples/src/test/java/org/apache/hama/examples/SpMVTest.java (added)
+++ hama/trunk/examples/src/test/java/org/apache/hama/examples/SpMVTest.java Wed Dec 12 12:09:48 2012
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples;
+
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Writable;
+import org.apache.hama.HamaConfiguration;
+import org.apache.hama.examples.util.DenseVectorWritable;
+import org.apache.hama.examples.util.SparseVectorWritable;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * This class is test cases for {@link SpMV}. It will contain simple hand
+ * calculated cases, and cases of different matrix and vector sizes given with
+ * help of {@link RandomMatrixGenerator}
+ */
+public class SpMVTest {
+
+ protected static final Log LOG = LogFactory.getLog(SpMVTest.class);
+
+ private HamaConfiguration conf;
+ private FileSystem fs;
+ private String baseDir;
+
+ @Before
+ public void prepare() throws IOException {
+ conf = new HamaConfiguration();
+ fs = FileSystem.get(conf);
+ baseDir = fs.getHomeDirectory().toString() + "/spmv";
+ }
+
+ /**
+ * Simple test of running spmv from {@link ExampleDriver}. You should specify
+ * paths.
+ */
+ @Test
+ public void runFromDriver() {
+ try {
+ String matrixPath = "";
+ String vectorPath = "";
+ String outputPath = "";
+ if (matrixPath.isEmpty() || vectorPath.isEmpty() || outputPath.isEmpty()) {
+ LOG.info("Please setup input path for vector and matrix and output path for result.");
+ return;
+ }
+ ExampleDriver.main(new String[] { "spmv", matrixPath, vectorPath,
+ outputPath, "4" });
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getLocalizedMessage());
+ }
+ }
+
+ /**
+ * Simple test. multiplying [1 0 6 0] [2] [38] [0 4 0 0] * [3] = [12] [0 2 3
+ * 0] [6] [24] [3 0 0 5] [1] [11]
+ */
+ @Test
+ public void simpleSpMVTest() {
+ try {
+ HamaConfiguration conf = new HamaConfiguration();
+ String testDir = "/simple/";
+ int size = 4;
+
+ // creating test matrix
+ HashMap<Integer, Writable> inputMatrix = new HashMap<Integer, Writable>();
+ SparseVectorWritable vector0 = new SparseVectorWritable();
+ vector0.setSize(size);
+ vector0.addCell(0, 1);
+ vector0.addCell(2, 6);
+ SparseVectorWritable vector1 = new SparseVectorWritable();
+ vector1.setSize(size);
+ vector1.addCell(1, 4);
+ SparseVectorWritable vector2 = new SparseVectorWritable();
+ vector2.setSize(size);
+ vector2.addCell(1, 2);
+ vector2.addCell(2, 3);
+ SparseVectorWritable vector3 = new SparseVectorWritable();
+ vector3.setSize(size);
+ vector3.addCell(0, 3);
+ vector3.addCell(3, 5);
+ inputMatrix.put(0, vector0);
+ inputMatrix.put(1, vector1);
+ inputMatrix.put(2, vector2);
+ inputMatrix.put(3, vector3);
+ String matrixPath = baseDir + testDir + "inputMatrix";
+ writeMatrix(matrixPath, conf, inputMatrix);
+
+ HashMap<Integer, Writable> inputVector = new HashMap<Integer, Writable>();
+ DenseVectorWritable vector = new DenseVectorWritable();
+ vector.setSize(size);
+ vector.addCell(0, 2);
+ vector.addCell(1, 3);
+ vector.addCell(2, 6);
+ vector.addCell(3, 1);
+ inputVector.put(0, vector);
+ String vectorPath = baseDir + testDir + "inputVector";
+ writeMatrix(vectorPath, conf, inputVector);
+
+ String outputPath = baseDir + testDir;
+ SpMV.main(new String[] { matrixPath, vectorPath, outputPath, "4" });
+
+ String resultPath = SpMV.getResultPath();
+ DenseVectorWritable result = new DenseVectorWritable();
+ SpMV.readFromFile(resultPath, result, conf);
+
+ double expected[] = { 38, 12, 24, 11 };
+ if (result.getSize() != size)
+ throw new Exception("Incorrect size of output vector");
+ for (int i = 0; i < result.getSize(); i++)
+ if ((result.get(i) - expected[i]) < 0.01)
+ expected[i] = 0;
+
+ for (int i = 0; i < expected.length; i++)
+ if (expected[i] != 0)
+ throw new Exception("Result doesn't meets expectations");
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getLocalizedMessage());
+ }
+ }
+
+ /**
+ * Simple test of spmv work with files. You should specify paths.
+ */
+ @Test
+ public void simpleSpMVTestFile() {
+ try {
+ int size = 4;
+ String matrixPath = "";
+ String vectorPath = "";
+ String outputPath = "";
+ if (matrixPath.isEmpty() || vectorPath.isEmpty() || outputPath.isEmpty()) {
+ LOG.info("Please setup input path for vector and matrix and output path for result, "
+ + "if you want to run this example");
+ return;
+ }
+ SpMV.main(new String[] { matrixPath, vectorPath, outputPath, "4" });
+
+ String resultPath = SpMV.getResultPath();
+ DenseVectorWritable result = new DenseVectorWritable();
+ SpMV.readFromFile(resultPath, result, conf);
+
+ double expected[] = { 38, 12, 24, 11 };
+ if (result.getSize() != size)
+ throw new Exception("Incorrect size of output vector");
+ for (int i = 0; i < result.getSize(); i++)
+ if ((result.get(i) - expected[i]) < 0.01)
+ expected[i] = 0;
+
+ for (int i = 0; i < expected.length; i++)
+ if (expected[i] != 0)
+ throw new Exception("Result doesn't meets expectations");
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getLocalizedMessage());
+ }
+ }
+
+ /**
+ * This method gives the ability to write matrix from memory to file. It
+ * should be used with small matrices and for test purposes only.
+ *
+ * @param pathString path to file where matrix will be writed.
+ * @param conf configuration
+ * @param matrix map of row indeces and values presented as Writable
+ * @throws IOException
+ */
+ public static void writeMatrix(String pathString, Configuration conf,
+ Map<Integer, Writable> matrix) throws IOException {
+ boolean inited = false;
+ FileSystem fs = FileSystem.get(conf);
+ SequenceFile.Writer writer = null;
+ try {
+ for (Integer index : matrix.keySet()) {
+ IntWritable key = new IntWritable(index);
+ Writable value = matrix.get(index);
+ if (!inited) {
+ writer = new SequenceFile.Writer(fs, conf, new Path(pathString),
+ IntWritable.class, value.getClass());
+ inited = true;
+ }
+ writer.append(key, value);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (writer != null)
+ writer.close();
+ }
+
+ }
+}