You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2010/06/22 02:18:11 UTC
svn commit: r956746 - in /incubator/hama/trunk: ./
src/examples/org/apache/hama/examples/
src/examples/org/apache/hama/examples/mapreduce/
src/java/org/apache/hama/io/ src/test/org/apache/hama/examples/
src/test/org/apache/hama/io/
Author: edwardyoon
Date: Tue Jun 22 00:18:09 2010
New Revision: 956746
URL: http://svn.apache.org/viewvc?rev=956746&view=rev
Log:
Moving BlockID, Pair classes to examples package
Added:
incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockID.java
incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/Pair.java
incubator/hama/trunk/src/test/org/apache/hama/examples/TestBlockID.java
Removed:
incubator/hama/trunk/src/java/org/apache/hama/io/BlockID.java
incubator/hama/trunk/src/java/org/apache/hama/io/Pair.java
incubator/hama/trunk/src/test/org/apache/hama/io/TestBlockID.java
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/src/examples/org/apache/hama/examples/JacobiEigen.java
incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java
incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java
incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java
incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java
incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java
incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java
incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotMap.java
incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java
Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Tue Jun 22 00:18:09 2010
@@ -43,6 +43,7 @@ Trunk (unreleased changes)
IMPROVEMENTS
+ HAMA-264: Moving BlockID, Pair classes to examples package (edwardyoon)
HAMA-269: Add getter for serverName to BSPPeerInterface (edwardyoon)
HAMA-257: Design BSP program code interface (edwardyoon)
HAMA-261: Remove meaningless unit tests (edwardyoon)
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/JacobiEigen.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/JacobiEigen.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/JacobiEigen.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/JacobiEigen.java Tue Jun 22 00:18:09 2010
@@ -24,10 +24,10 @@ import org.apache.hama.HamaAdmin;
import org.apache.hama.HamaAdminImpl;
import org.apache.hama.examples.mapreduce.DummyMapper;
import org.apache.hama.examples.mapreduce.JacobiInitMap;
+import org.apache.hama.examples.mapreduce.Pair;
import org.apache.hama.examples.mapreduce.PivotInputFormat;
import org.apache.hama.examples.mapreduce.PivotMap;
import org.apache.hama.examples.mapreduce.RotationInputFormat;
-import org.apache.hama.io.Pair;
import org.apache.hama.matrix.DenseMatrix;
import org.apache.hama.matrix.Matrix;
import org.apache.hama.util.BytesUtil;
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/MatrixMultiplication.java Tue Jun 22 00:18:09 2010
@@ -37,12 +37,12 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hama.Constants;
import org.apache.hama.HamaAdmin;
import org.apache.hama.HamaAdminImpl;
+import org.apache.hama.examples.mapreduce.BlockID;
import org.apache.hama.examples.mapreduce.BlockMultMap;
import org.apache.hama.examples.mapreduce.BlockMultReduce;
import org.apache.hama.examples.mapreduce.CollectBlocksMapper;
import org.apache.hama.examples.mapreduce.DenseMatrixVectorMultMap;
import org.apache.hama.examples.mapreduce.DenseMatrixVectorMultReduce;
-import org.apache.hama.io.BlockID;
import org.apache.hama.matrix.DenseMatrix;
import org.apache.hama.matrix.Matrix;
import org.apache.hama.util.RandomVariable;
Added: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockID.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockID.java?rev=956746&view=auto
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockID.java (added)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockID.java Tue Jun 22 00:18:09 2010
@@ -0,0 +1,150 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.log4j.Logger;
+
+/** A WritableComparable for BlockIDs. */
+@SuppressWarnings("unchecked")
+public class BlockID implements WritableComparable {
+ static final Logger LOG = Logger.getLogger(BlockID.class);
+ public static final int PAD_SIZE = 15;
+ private int row;
+ private int column;
+ private int seq = -1;
+
+ public BlockID() {
+ }
+
+ public BlockID(int row, int column) {
+ set(row, column);
+ }
+
+ public BlockID(byte[] bytes) throws IOException {
+ String rKey = Bytes.toString(bytes);
+ String keys[] = null;
+ if (rKey.substring(0, 8).equals("00000000")) {
+ int i = 8;
+ while (rKey.charAt(i) == '0') {
+ i++;
+ }
+ keys = rKey.substring(i, rKey.length()).split("[,]");
+ } else {
+ int i = 0;
+ while (rKey.charAt(i) == '0') {
+ i++;
+ }
+ keys = rKey.substring(i, rKey.length()).split("[,]");
+ }
+
+ try {
+ this.row = Integer.parseInt(keys[1]);
+ String[] columns = keys[2].split("[-]");
+ this.column = Integer.parseInt(columns[0]);
+ } catch (ArrayIndexOutOfBoundsException e) {
+ throw new ArrayIndexOutOfBoundsException(rKey + "\n" + e);
+ }
+ }
+
+ public BlockID(int row, int column, int seq) {
+ set(row, column);
+ this.seq = seq;
+ }
+
+ public void set(int row, int column) {
+ this.row = row;
+ this.column = column;
+ }
+
+ public int getRow() {
+ return row;
+ }
+
+ public int getColumn() {
+ return column;
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ BlockID value = new BlockID(Bytes.readByteArray(in));
+ this.row = value.getRow();
+ this.column = value.getColumn();
+ }
+
+ public void write(DataOutput out) throws IOException {
+ Bytes.writeByteArray(out, Bytes.toBytes(this.toString()));
+ }
+
+ /**
+ * Make BlockID's string representation be same format.
+ */
+ public String toString() {
+ int zeros = PAD_SIZE - String.valueOf(row).length()
+ - String.valueOf(column).length();
+ StringBuffer buf = new StringBuffer();
+ for (int i = 0; i < zeros; ++i) {
+ buf.append("0");
+ }
+
+ if(seq > -1) {
+ return buf.toString() + "," + row + "," + column + "-" + seq;
+ } else {
+ return buf.toString() + "," + row + "," + column;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ // simply use a prime number
+ // may be need a more balance hash function
+ return row * 37 + column;
+ }
+
+ public int compareTo(Object o) {
+ int thisRow = this.row;
+ int thatRow = ((BlockID) o).row;
+ int thisColumn = this.column;
+ int thatColumn = ((BlockID) o).column;
+
+ if (thisRow != thatRow) {
+ return (thisRow < thatRow ? -1 : 1);
+ } else {
+ return (thisColumn < thatColumn ? -1 : (thisColumn == thatColumn ? 0 : 1));
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null)
+ return false;
+ if (!(o instanceof BlockID))
+ return false;
+ return compareTo(o) == 0;
+ }
+
+ public byte[] getBytes() {
+ return Bytes.toBytes(this.toString());
+ }
+}
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultMap.java Tue Jun 22 00:18:09 2010
@@ -8,7 +8,6 @@ import org.apache.hadoop.hbase.mapreduce
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hama.Constants;
-import org.apache.hama.io.BlockID;
import org.apache.hama.matrix.SubMatrix;
public class BlockMultMap extends TableMapper<BlockID, BytesWritable> {
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/BlockMultReduce.java Tue Jun 22 00:18:09 2010
@@ -9,7 +9,6 @@ import org.apache.hadoop.hbase.util.Byte
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hama.Constants;
-import org.apache.hama.io.BlockID;
import org.apache.hama.matrix.SubMatrix;
import org.apache.hama.util.BytesUtil;
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksMapper.java Tue Jun 22 00:18:09 2010
@@ -8,7 +8,6 @@ import org.apache.hadoop.hbase.client.Re
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.io.MapWritable;
-import org.apache.hama.io.BlockID;
import org.apache.hama.matrix.DenseVector;
import org.apache.hama.util.BytesUtil;
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/CollectBlocksReducer.java Tue Jun 22 00:18:09 2010
@@ -11,7 +11,6 @@ import org.apache.hadoop.hbase.util.Byte
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hama.Constants;
-import org.apache.hama.io.BlockID;
import org.apache.hama.matrix.DenseVector;
import org.apache.hama.matrix.SubMatrix;
Added: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/Pair.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/Pair.java?rev=956746&view=auto
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/Pair.java (added)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/Pair.java Tue Jun 22 00:18:09 2010
@@ -0,0 +1,65 @@
+package org.apache.hama.examples.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.WritableComparable;
+
+/** A Pair stands for (row, column) pair **/
+public class Pair implements WritableComparable<Pair> {
+
+ int row, col;
+
+ public Pair() {}
+
+ public Pair(int row_, int col_) {
+ set(row_, col_);
+ }
+
+ public int getRow() { return row; }
+ public int getColumn() { return col; }
+
+ public void setRow(int row_) { row = row_; }
+ public void setColumn(int col_) { col = col_; }
+ public void set(int row_, int col_) {
+ row = row_;
+ col = col_;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ row = in.readInt();
+ col = in.readInt();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(row);
+ out.writeInt(col);
+ }
+
+ @Override
+ public int compareTo(Pair p) {
+ return row == p.row ? col - p.col : row - p.row;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ Pair pair = (Pair)obj;
+ return compareTo(pair) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return row;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append('(').append(row).append(',').append(col).append(')');
+ return sb.toString();
+ }
+
+}
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotInputFormat.java Tue Jun 22 00:18:09 2010
@@ -30,7 +30,6 @@ import org.apache.hadoop.mapreduce.Recor
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.StringUtils;
import org.apache.hama.examples.JacobiEigen;
-import org.apache.hama.io.Pair;
import org.apache.hama.util.BytesUtil;
public class PivotInputFormat extends InputFormat<Pair, DoubleWritable>
Modified: incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotMap.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotMap.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotMap.java (original)
+++ incubator/hama/trunk/src/examples/org/apache/hama/examples/mapreduce/PivotMap.java Tue Jun 22 00:18:09 2010
@@ -4,7 +4,6 @@ import java.io.IOException;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hama.io.Pair;
public class PivotMap extends
Mapper<Pair, DoubleWritable, Pair, DoubleWritable> {
Added: incubator/hama/trunk/src/test/org/apache/hama/examples/TestBlockID.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/examples/TestBlockID.java?rev=956746&view=auto
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/examples/TestBlockID.java (added)
+++ incubator/hama/trunk/src/test/org/apache/hama/examples/TestBlockID.java Tue Jun 22 00:18:09 2010
@@ -0,0 +1,69 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hama.examples;
+
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hama.examples.mapreduce.BlockID;
+
+public class TestBlockID extends TestCase {
+ final static Log LOG = LogFactory.getLog(TestBlockID.class.getName());
+
+ /**
+ * BlockID object compare
+ */
+ public void testCompare() {
+ BlockID a = new BlockID(1, 3);
+ BlockID b = new BlockID(1, 1);
+ assertEquals(a.compareTo(b), 1);
+
+ BlockID c = new BlockID(3, 1);
+ BlockID d = new BlockID(1, 1);
+ assertEquals(a.compareTo(c), -1);
+
+ assertEquals(b.compareTo(d), 0);
+ }
+
+ /**
+ * BlockID object IO
+ * @throws IOException
+ */
+ public void testIO() throws IOException {
+ DataOutputBuffer outBuf = new DataOutputBuffer();
+ DataInputBuffer inBuf = new DataInputBuffer();
+
+ BlockID a = new BlockID(1, 3);
+ outBuf.reset();
+ a.write(outBuf);
+
+ inBuf.reset(outBuf.getData(), outBuf.getLength());
+ BlockID b = new BlockID();
+ b.readFields(inBuf);
+
+ assertEquals(0, a.compareTo(b));
+ }
+
+}
Modified: incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java?rev=956746&r1=956745&r2=956746&view=diff
==============================================================================
--- incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java (original)
+++ incubator/hama/trunk/src/test/org/apache/hama/examples/TestFileMatrixBlockMult.java Tue Jun 22 00:18:09 2010
@@ -28,9 +28,9 @@ import org.apache.hadoop.mapreduce.lib.i
import org.apache.hama.Constants;
import org.apache.hama.HamaCluster;
import org.apache.hama.HamaConfiguration;
+import org.apache.hama.examples.mapreduce.BlockID;
import org.apache.hama.examples.mapreduce.BlockMultMap;
import org.apache.hama.examples.mapreduce.BlockMultReduce;
-import org.apache.hama.io.BlockID;
import org.apache.hama.matrix.DenseMatrix;
import org.apache.hama.matrix.DenseVector;
import org.apache.hama.matrix.Matrix;
Re: svn commit: r956746 - in /incubator/hama/trunk: ./ src/examples/org/apache/hama/examples/ src/examples/org/apache/hama/examples/mapreduce/ src/java/org/apache/hama/io/ src/test/org/apache/hama/examples/ src/test/org/apache/hama/io/
Posted by Mari <ma...@hotmail.com>.
Enviado.
El 24/06/2010, a las 20:56, Mari <ma...@hotmail.com> escribió:
>
>
> Enviado.
>
>
> El 21/06/2010, a las 21:18, edwardyoon@apache.org escribió:
>
>
Re: svn commit: r956746 - in /incubator/hama/trunk: ./ src/examples/org/apache/hama/examples/ src/examples/org/apache/hama/examples/mapreduce/ src/java/org/apache/hama/io/ src/test/org/apache/hama/examples/ src/test/org/apache/hama/io/
Posted by Mari <ma...@hotmail.com>.
Enviado.
El 21/06/2010, a las 21:18, edwardyoon@apache.org escribió:
Re: svn commit: r956746 - in /incubator/hama/trunk: ./ src/examples/org/apache/hama/examples/ src/examples/org/apache/hama/examples/mapreduce/ src/java/org/apache/hama/io/ src/test/org/apache/hama/examples/ src/test/org/apache/hama/io/
Posted by Mari <ma...@hotmail.com>.
Enviado.
El 21/06/2010, a las 21:18, edwardyoon@apache.org escribió: