You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2011/03/31 11:25:30 UTC
svn commit: r1087225 [5/7] - in /mahout/trunk:
core/src/main/java/org/apache/mahout/cf/taste/hadoop/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/
core/src/main/java/org/apache/m...
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDPrototype.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDPrototype.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDPrototype.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDPrototype.java Thu Mar 31 09:25:25 2011
@@ -20,11 +20,12 @@ package org.apache.mahout.math.hadoop.st
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import org.apache.mahout.common.RandomUtils;
+import org.apache.mahout.common.iterator.CopyConstructorIterator;
import org.apache.mahout.math.DenseMatrix;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.SingularValueDecomposition;
@@ -34,26 +35,21 @@ import org.apache.mahout.math.ssvd.Eigen
/**
* SSVD protoptype: non-MR concept verification for Givens QR & SSVD basic
* algorithms
- *
- * @author Dmitriy
- *
*/
public class SSVDPrototype {
-
- private Omega omega;
- private int kp; // k+p
+ private final Omega omega;
+ private final int kp; // k+p
private GivensThinSolver qSolver;
- private double[] yRow;
+ private final double[] yRow;
private int cnt;
private int blckCnt;
- private int r;
- private List<UpperTriangular> rBlocks = new ArrayList<UpperTriangular>();
- private List<double[][]> qtBlocks = new ArrayList<double[][]>();
- private List<double[]> yLookahead;
+ private final int r;
+ private final List<UpperTriangular> rBlocks = new ArrayList<UpperTriangular>();
+ private final List<double[][]> qtBlocks = new ArrayList<double[][]>();
+ private final List<double[]> yLookahead;
public SSVDPrototype(long seed, int kp, int r) {
- super();
this.kp = kp;
omega = new Omega(seed, kp / 2, kp - (kp / 2));
yRow = new double[kp];
@@ -62,7 +58,7 @@ public class SSVDPrototype {
yLookahead = new ArrayList<double[]>(kp);
}
- void firstPass(int aRowId, Vector aRow) throws IOException {
+ void firstPass(Vector aRow) {
omega.computeYRow(aRow, yRow);
@@ -72,8 +68,9 @@ public class SSVDPrototype {
while (yLookahead.size() > kp) {
- if (qSolver == null)
+ if (qSolver == null) {
qSolver = new GivensThinSolver(r, kp);
+ }
qSolver.appendRow(yLookahead.remove(0));
if (qSolver.isFull()) {
@@ -90,14 +87,16 @@ public class SSVDPrototype {
void finishFirstPass() {
- if (qSolver == null && yLookahead.size() == 0)
+ if (qSolver == null && yLookahead.isEmpty()) {
return;
- if (qSolver == null)
+ }
+ if (qSolver == null) {
qSolver = new GivensThinSolver(yLookahead.size(), kp);
+ }
// grow q solver up if necessary
qSolver.adjust(qSolver.getCnt() + yLookahead.size());
- while (yLookahead.size() > 0) {
+ while (!yLookahead.isEmpty()) {
qSolver.appendRow(yLookahead.remove(0));
if (qSolver.isFull()) {
@@ -111,14 +110,16 @@ public class SSVDPrototype {
}
// simulate reducers -- produce qHats
- for (int i = 0; i < rBlocks.size(); i++)
- qtBlocks.set(i, GivensThinSolver.computeQtHat(qtBlocks.get(i), i,
- new DeepCopyIterator(rBlocks.listIterator())));
+ for (int i = 0; i < rBlocks.size(); i++) {
+ qtBlocks.set(i,
+ GivensThinSolver.computeQtHat(qtBlocks.get(i), i,
+ new CopyConstructorIterator<UpperTriangular>(rBlocks.listIterator())));
+ }
cnt = 0;
blckCnt = 0;
}
- void secondPass(int aRowId, Vector aRow, PartialRowEmitter btEmitter)
+ void secondPass(Vector aRow, PartialRowEmitter btEmitter)
throws IOException {
int n = aRow.size();
double[][] qtHat = qtBlocks.get(blckCnt);
@@ -127,8 +128,9 @@ public class SSVDPrototype {
int qRowBlckIndex = r - cnt - 1; // <-- reverse order since we fed A in
// reverse
double[] qRow = new double[kp];
- for (int i = 0; i < kp; i++)
+ for (int i = 0; i < kp; i++) {
qRow[i] = qtHat[i][qRowBlckIndex];
+ }
Vector qRowV = new DenseVector(qRow, true);
if (++cnt == r) {
@@ -136,37 +138,14 @@ public class SSVDPrototype {
cnt = 0;
}
- for (int i = 0; i < n; i++)
+ for (int i = 0; i < n; i++) {
btEmitter.emitRow(i, qRowV.times(aRow.getQuick(i)));
-
- }
-
- private static class DeepCopyIterator implements Iterator<UpperTriangular> {
-
- private Iterator<UpperTriangular> delegate;
-
- public DeepCopyIterator(Iterator<UpperTriangular> del) {
- super();
- delegate = del;
- }
-
- public boolean hasNext() {
- return delegate.hasNext();
- }
-
- public UpperTriangular next() {
-
- return new UpperTriangular(delegate.next());
- }
-
- public void remove() {
- delegate.remove();
}
}
- public static void testThinQr(int dims, int kp, final long rndSeed)
- throws Exception {
+
+ public static void testThinQr(int dims, int kp) throws IOException {
DenseMatrix mx = new DenseMatrix(dims << 2, dims);
// mx.assign(new UnaryFunction() {
@@ -179,10 +158,12 @@ public class SSVDPrototype {
// }
// });
- Random rnd = new Random();
- for (int i = 0; i < mx.rowSize(); i++)
- for (int j = 0; j < mx.columnSize(); j++)
+ Random rnd = RandomUtils.getRandom();
+ for (int i = 0; i < mx.rowSize(); i++) {
+ for (int j = 0; j < mx.columnSize(); j++) {
mx.set(i, j, rnd.nextDouble() * 1000);
+ }
+ }
mx.setQuick(0, 0, 1);
mx.setQuick(0, 1, 2);
@@ -197,34 +178,38 @@ public class SSVDPrototype {
SingularValueDecomposition svd2 = new SingularValueDecomposition(mx);
double[] svaluesControl = svd2.getSingularValues();
- for (int i = 0; i < kp; i++)
+ for (int i = 0; i < kp; i++) {
System.out.printf("%.3e ", svaluesControl[i]);
+ }
System.out.println();
int m = mx.rowSize(); /* ,n=mx.columnSize(); */
- long seed = new Random().nextLong();
+ long seed = RandomUtils.getRandom().nextLong();
final Map<Integer, Vector> btRows = new HashMap<Integer, Vector>();
PartialRowEmitter btEmitter = new PartialRowEmitter() {
@Override
- public void emitRow(int rowNum, Vector row) throws IOException {
+ public void emitRow(int rowNum, Vector row) {
Vector btRow = btRows.get(rowNum);
- if (btRow != null)
+ if (btRow != null) {
row.addTo(btRow);
+ }
btRows.put(rowNum, btRow == null ? new DenseVector(row) : btRow);
}
};
SSVDPrototype mapperSimulation = new SSVDPrototype(seed, kp, 3000);
- for (int i = 0; i < m; i++)
- mapperSimulation.firstPass(i, mx.getRow(i));
+ for (int i = 0; i < m; i++) {
+ mapperSimulation.firstPass(mx.getRow(i));
+ }
mapperSimulation.finishFirstPass();
- for (int i = 0; i < m; i++)
- mapperSimulation.secondPass(i, mx.getRow(i), btEmitter);
+ for (int i = 0; i < m; i++) {
+ mapperSimulation.secondPass(mx.getRow(i), btEmitter);
+ }
// LocalSSVDTest.assertOrthonormality(mapperSimulation.m_qt.transpose(),
// false,1e-10);
@@ -234,10 +219,11 @@ public class SSVDPrototype {
PartialRowEmitter bbtEmitter = new PartialRowEmitter() {
@Override
- public void emitRow(int rowNum, Vector row) throws IOException {
+ public void emitRow(int rowNum, Vector row) {
Vector bbtRow = bbt.get(rowNum);
- if (bbtRow != null)
+ if (bbtRow != null) {
row.addTo(bbtRow);
+ }
bbt.put(rowNum, bbtRow == null ? new DenseVector(row) : bbtRow);
}
};
@@ -245,32 +231,35 @@ public class SSVDPrototype {
for (Map.Entry<Integer, Vector> btRowEntry : btRows.entrySet()) {
Vector btRow = btRowEntry.getValue();
assert btRow.size() == kp;
- for (int i = 0; i < kp; i++)
+ for (int i = 0; i < kp; i++) {
bbtEmitter.emitRow(i, btRow.times(btRow.getQuick(i)));
+ }
}
double[][] bbtValues = new double[kp][];
for (int i = 0; i < kp; i++) {
bbtValues[i] = new double[kp];
Vector bbtRow = bbt.get(i);
- for (int j = 0; j < kp; j++)
+ for (int j = 0; j < kp; j++) {
bbtValues[i][j] = bbtRow.getQuick(j);
+ }
}
EigenSolverWrapper eigenWrapper = new EigenSolverWrapper(bbtValues);
double[] eigenva2 = eigenWrapper.getEigenValues();
double[] svalues = new double[kp];
- for (int i = 0; i < kp; i++)
+ for (int i = 0; i < kp; i++) {
svalues[i] = Math.sqrt(eigenva2[i]); // sqrt?
+ }
- for (int i = 0; i < kp; i++)
+ for (int i = 0; i < kp; i++) {
System.out.printf("%.3e ", svalues[i]);
+ }
System.out.println();
}
- public static void testBlockQrWithSSVD(int dims, int kp, int r,
- final long rndSeed) throws Exception {
+ public static void testBlockQrWithSSVD(int dims, int kp, int r, long rndSeed) throws IOException {
DenseMatrix mx = new DenseMatrix(dims << 2, dims);
// mx.assign(new UnaryFunction() {
@@ -283,10 +272,12 @@ public class SSVDPrototype {
// }
// });
- Random rnd = new Random();
- for (int i = 0; i < mx.rowSize(); i++)
- for (int j = 0; j < mx.columnSize(); j++)
+ Random rnd = RandomUtils.getRandom();
+ for (int i = 0; i < mx.rowSize(); i++) {
+ for (int j = 0; j < mx.columnSize(); j++) {
mx.set(i, j, (rnd.nextDouble() - 0.5) * 1000);
+ }
+ }
mx.setQuick(0, 0, 1);
mx.setQuick(0, 1, 2);
mx.setQuick(0, 2, 3);
@@ -300,8 +291,9 @@ public class SSVDPrototype {
SingularValueDecomposition svd2 = new SingularValueDecomposition(mx);
double[] svaluesControl = svd2.getSingularValues();
- for (int i = 0; i < kp; i++)
+ for (int i = 0; i < kp; i++) {
System.out.printf("%e ", svaluesControl[i]);
+ }
System.out.println();
int m = mx.rowSize(); /* ,n=mx.columnSize(); */
@@ -310,22 +302,25 @@ public class SSVDPrototype {
PartialRowEmitter btEmitter = new PartialRowEmitter() {
@Override
- public void emitRow(int rowNum, Vector row) throws IOException {
+ public void emitRow(int rowNum, Vector row) {
Vector btRow = btRows.get(rowNum);
- if (btRow != null)
+ if (btRow != null) {
row.addTo(btRow);
+ }
btRows.put(rowNum, btRow == null ? new DenseVector(row) : btRow);
}
};
SSVDPrototype mapperSimulation = new SSVDPrototype(rndSeed, kp, r);
- for (int i = 0; i < m; i++)
- mapperSimulation.firstPass(i, mx.getRow(i));
+ for (int i = 0; i < m; i++) {
+ mapperSimulation.firstPass(mx.getRow(i));
+ }
mapperSimulation.finishFirstPass();
- for (int i = 0; i < m; i++)
- mapperSimulation.secondPass(i, mx.getRow(i), btEmitter);
+ for (int i = 0; i < m; i++) {
+ mapperSimulation.secondPass(mx.getRow(i), btEmitter);
+ }
// LocalSSVDTest.assertOrthonormality(mapperSimulation.m_qt.transpose(),
// false,1e-10);
@@ -335,10 +330,11 @@ public class SSVDPrototype {
PartialRowEmitter bbtEmitter = new PartialRowEmitter() {
@Override
- public void emitRow(int rowNum, Vector row) throws IOException {
+ public void emitRow(int rowNum, Vector row) {
Vector bbtRow = bbt.get(rowNum);
- if (bbtRow != null)
+ if (bbtRow != null) {
row.addTo(bbtRow);
+ }
bbt.put(rowNum, bbtRow == null ? new DenseVector(row) : bbtRow);
}
};
@@ -346,33 +342,37 @@ public class SSVDPrototype {
for (Map.Entry<Integer, Vector> btRowEntry : btRows.entrySet()) {
Vector btRow = btRowEntry.getValue();
assert btRow.size() == kp;
- for (int i = 0; i < kp; i++)
+ for (int i = 0; i < kp; i++) {
bbtEmitter.emitRow(i, btRow.times(btRow.getQuick(i)));
+ }
}
double[][] bbtValues = new double[kp][];
for (int i = 0; i < kp; i++) {
bbtValues[i] = new double[kp];
Vector bbtRow = bbt.get(i);
- for (int j = 0; j < kp; j++)
+ for (int j = 0; j < kp; j++) {
bbtValues[i][j] = bbtRow.getQuick(j);
+ }
}
EigenSolverWrapper eigenWrapper = new EigenSolverWrapper(bbtValues);
double[] eigenva2 = eigenWrapper.getEigenValues();
double[] svalues = new double[kp];
- for (int i = 0; i < kp; i++)
+ for (int i = 0; i < kp; i++) {
svalues[i] = Math.sqrt(eigenva2[i]); // sqrt?
+ }
- for (int i = 0; i < kp; i++)
+ for (int i = 0; i < kp; i++) {
System.out.printf("%e ", svalues[i]);
+ }
System.out.println();
}
public static void main(String[] args) throws Exception {
// testThinQr();
- long seed = new Random().nextLong();
+ long seed = RandomUtils.getRandom().nextLong();
testBlockQrWithSSVD(200, 200, 800, seed);
testBlockQrWithSSVD(200, 20, 800, seed);
testBlockQrWithSSVD(200, 20, 850, seed); // test trimming
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java Thu Mar 31 09:25:25 2011
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.Deque;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
@@ -37,6 +38,8 @@ import org.apache.hadoop.io.SequenceFile
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Writable;
import org.apache.mahout.common.IOUtils;
+import org.apache.mahout.common.RandomUtils;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileValueIterable;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
@@ -86,9 +89,6 @@ import org.apache.mahout.math.ssvd.Eigen
* containing n x k V matrix file(s).
*
* </UL>
- *
- *
- *
*/
public class SSVDSolver {
@@ -99,13 +99,13 @@ public class SSVDSolver {
private String vPath;
// configured stuff
- private Configuration conf;
- private Path[] inputPath;
- private Path outputPath;
- private int ablockRows;
- private int k;
- private int p;
- private int reduceTasks;
+ private final Configuration conf;
+ private final Path[] inputPath;
+ private final Path outputPath;
+ private final int ablockRows;
+ private final int k;
+ private final int p;
+ private final int reduceTasks;
private int minSplitSize = -1;
private boolean cUHalfSigma;
private boolean cVHalfSigma;
@@ -135,9 +135,13 @@ public class SSVDSolver {
* @throws IOException
* when IO condition occurs.
*/
-
- public SSVDSolver(Configuration conf, Path[] inputPath, Path outputPath,
- int ablockRows, int k, int p, int reduceTasks) throws IOException {
+ public SSVDSolver(Configuration conf,
+ Path[] inputPath,
+ Path outputPath,
+ int ablockRows,
+ int k,
+ int p,
+ int reduceTasks) {
this.conf = conf;
this.inputPath = inputPath;
this.outputPath = outputPath;
@@ -235,10 +239,9 @@ public class SSVDSolver {
*/
public void run() throws IOException {
- LinkedList<Closeable> closeables = new LinkedList<Closeable>();
+ Deque<Closeable> closeables = new LinkedList<Closeable>();
try {
- Class<? extends Writable> labelType = sniffInputLabelType(inputPath,
- conf, closeables);
+ Class<? extends Writable> labelType = sniffInputLabelType(inputPath, conf);
FileSystem fs = FileSystem.get(conf);
Path qPath = new Path(outputPath, "Q-job");
@@ -253,30 +256,30 @@ public class SSVDSolver {
fs.delete(outputPath, true);
}
- Random rnd = new Random();
+ Random rnd = RandomUtils.getRandom();
long seed = rnd.nextLong();
- QJob.run(conf, inputPath, qPath, ablockRows, minSplitSize, k, p, seed,
- reduceTasks);
+ QJob.run(conf, inputPath, qPath, ablockRows, minSplitSize, k, p, seed, reduceTasks);
- BtJob.run(conf, inputPath, qPath, btPath, minSplitSize, k, p,
- reduceTasks, labelType);
+ BtJob.run(conf, inputPath, qPath, btPath, minSplitSize, k, p, reduceTasks, labelType);
BBtJob.run(conf, new Path(btPath, BtJob.OUTPUT_BT + "-*"), bbtPath, 1);
- UpperTriangular bbt = loadUpperTriangularMatrix(fs, new Path(bbtPath,
- BBtJob.OUTPUT_BBT + "-*"), conf);
+ UpperTriangular bbt = loadUpperTriangularMatrix(fs, new Path(bbtPath, BBtJob.OUTPUT_BBT + "-*"), conf);
// convert bbt to something our eigensolver could understand
assert bbt.columnSize() == k + p;
double[][] bbtSquare = new double[k + p][];
- for (int i = 0; i < k + p; i++)
+ for (int i = 0; i < k + p; i++) {
bbtSquare[i] = new double[k + p];
+ }
- for (int i = 0; i < k + p; i++)
- for (int j = i; j < k + p; j++)
+ for (int i = 0; i < k + p; i++) {
+ for (int j = i; j < k + p; j++) {
bbtSquare[i][j] = bbtSquare[j][i] = bbt.getQuick(i, j);
+ }
+ }
svalues = new double[k + p];
@@ -284,8 +287,9 @@ public class SSVDSolver {
EigenSolverWrapper eigenWrapper = new EigenSolverWrapper(bbtSquare);
double[] eigenva2 = eigenWrapper.getEigenValues();
- for (int i = 0; i < k + p; i++)
+ for (int i = 0; i < k + p; i++) {
svalues[i] = Math.sqrt(eigenva2[i]); // sqrt?
+ }
// save/redistribute UHat
//
@@ -322,17 +326,21 @@ public class SSVDSolver {
svWriter.close();
UJob ujob = null;
+ if (computeU) {
+ ujob = new UJob();
+ ujob.start(conf,
+ new Path(btPath, BtJob.OUTPUT_Q + "-*"),
+ uHatPath, svPath, uPath, k, reduceTasks, labelType, cUHalfSigma);
+ // actually this is map-only job anyway
+ }
+
VJob vjob = null;
- if (computeU)
- (ujob = new UJob()).start(conf,
- new Path(btPath, BtJob.OUTPUT_Q + "-*"), uHatPath, svPath, uPath,
- k, reduceTasks, labelType, cUHalfSigma); // actually this is
- // map-only job anyway
-
- if (computeV)
- (vjob = new VJob()).start(conf,
- new Path(btPath, BtJob.OUTPUT_BT + "-*"), uHatPath, svPath, vPath,
- k, reduceTasks, cVHalfSigma);
+ if (computeV) {
+ vjob = new VJob();
+ vjob.start(conf,
+ new Path(btPath, BtJob.OUTPUT_BT + "-*"),
+ uHatPath, svPath, vPath, k, reduceTasks, cVHalfSigma);
+ }
if (ujob != null) {
ujob.waitForCompletion();
@@ -354,49 +362,40 @@ public class SSVDSolver {
}
- private static Class<? extends Writable> sniffInputLabelType(
- Path[] inputPath, Configuration conf, LinkedList<Closeable> closeables)
+ private static Class<? extends Writable> sniffInputLabelType(Path[] inputPath, Configuration conf)
throws IOException {
FileSystem fs = FileSystem.get(conf);
for (Path p : inputPath) {
FileStatus[] fstats = fs.globStatus(p);
- if (fstats == null || fstats.length == 0)
+ if (fstats == null || fstats.length == 0) {
continue;
- SequenceFile.Reader r = new SequenceFile.Reader(fs, fstats[0].getPath(),
- conf);
- closeables.addFirst(r);
-
+ }
+ SequenceFile.Reader r = new SequenceFile.Reader(fs, fstats[0].getPath(), conf);
try {
return r.getKeyClass().asSubclass(Writable.class);
} finally {
- closeables.remove(r);
r.close();
}
}
-
- throw new IOException(
- "Unable to open input files to determine input label type.");
+ throw new IOException("Unable to open input files to determine input label type.");
}
- private static final Pattern OUTPUT_FILE_PATTERN = Pattern
- .compile("(\\w+)-(m|r)-(\\d+)(\\.\\w+)?");
+ private static final Pattern OUTPUT_FILE_PATTERN = Pattern.compile("(\\w+)-(m|r)-(\\d+)(\\.\\w+)?");
- static Comparator<FileStatus> partitionComparator = new Comparator<FileStatus>() {
- private Matcher matcher = OUTPUT_FILE_PATTERN.matcher("");
+ static final Comparator<FileStatus> partitionComparator = new Comparator<FileStatus>() {
+ private final Matcher matcher = OUTPUT_FILE_PATTERN.matcher("");
@Override
public int compare(FileStatus o1, FileStatus o2) {
matcher.reset(o1.getPath().getName());
- if (!matcher.matches())
- throw new RuntimeException(String.format(
- "Unexpected file name, unable to deduce partition #:%s", o1
- .getPath().toString()));
+ if (!matcher.matches()) {
+ throw new IllegalArgumentException("Unexpected file name, unable to deduce partition #:" + o1.getPath());
+ }
int p1 = Integer.parseInt(matcher.group(3));
matcher.reset(o2.getPath().getName());
- if (!matcher.matches())
- throw new RuntimeException(String.format(
- "Unexpected file name, unable to deduce partition #:%s", o2
- .getPath().toString()));
+ if (!matcher.matches()) {
+ throw new IllegalArgumentException("Unexpected file name, unable to deduce partition #:" + o2.getPath());
+ }
int p2 = Integer.parseInt(matcher.group(3));
return p1 - p2;
@@ -418,16 +417,14 @@ public class SSVDSolver {
* @throws IOException
* when I/O occurs.
*/
- public static double[][] loadDistributedRowMatrix(FileSystem fs, Path glob,
- Configuration conf) throws IOException {
+ public static double[][] loadDistributedRowMatrix(FileSystem fs, Path glob, Configuration conf) throws IOException {
FileStatus[] files = fs.globStatus(glob);
- if (files == null)
+ if (files == null) {
return null;
+ }
List<double[]> denseData = new ArrayList<double[]>();
- IntWritable iw = new IntWritable();
- VectorWritable vw = new VectorWritable();
// int m=0;
@@ -436,22 +433,15 @@ public class SSVDSolver {
Arrays.sort(files, partitionComparator);
for (FileStatus fstat : files) {
- Path file = fstat.getPath();
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
- try {
- while (reader.next(iw, vw)) {
- Vector v = vw.get();
- int size;
- double[] row = new double[size = v.size()];
- for (int i = 0; i < size; i++)
- row[i] = v.get(i);
- // ignore row label.
- // int rowIndex=iw.get();
- denseData.add(row);
-
+ for (VectorWritable value : new SequenceFileValueIterable<VectorWritable>(fstat.getPath(), true, conf)) {
+ Vector v = value.get();
+ int size = v.size();
+ double[] row = new double[size];
+ for (int i = 0; i < size; i++) {
+ row[i] = v.get(i);
}
- } finally {
- reader.close();
+ // ignore row label.
+ denseData.add(row);
}
}
@@ -462,37 +452,29 @@ public class SSVDSolver {
Path glob, Configuration conf) throws IOException {
FileStatus[] files = fs.globStatus(glob);
- if (files == null)
+ if (files == null) {
return null;
-
- IntWritable iw = new IntWritable();
- VectorWritable vw = new VectorWritable();
- UpperTriangular result = null;
+ }
// assume it is partitioned output, so we need to read them up
// in order of partitions.
Arrays.sort(files, partitionComparator);
+ UpperTriangular result = null;
for (FileStatus fstat : files) {
- Path file = fstat.getPath();
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
- try {
- while (reader.next(iw, vw)) {
- Vector v = vw.get();
- if (result == null)
- result = new UpperTriangular(v);
- else
- throw new IOException(
- "Unexpected overrun in upper triangular matrix files");
+ for (VectorWritable value : new SequenceFileValueIterable<VectorWritable>(fstat.getPath(), true, conf)) {
+ Vector v = value.get();
+ if (result == null) {
+ result = new UpperTriangular(v);
+ } else {
+ throw new IOException("Unexpected overrun in upper triangular matrix files");
}
- } finally {
- reader.close();
}
}
- if (result == null)
- throw new IOException(
- "Unexpected underrun in upper triangular matrix files");
+ if (result == null) {
+ throw new IOException("Unexpected underrun in upper triangular matrix files");
+ }
return result;
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java Thu Mar 31 09:25:25 2011
@@ -20,15 +20,14 @@ package org.apache.mahout.vectorizer;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
@@ -41,7 +40,10 @@ import org.apache.hadoop.mapreduce.lib.i
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.mahout.common.HadoopUtil;
+import org.apache.mahout.common.Pair;
import org.apache.mahout.common.StringTuple;
+import org.apache.mahout.common.iterator.sequencefile.PathType;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileDirIterable;
import org.apache.mahout.math.VectorWritable;
import org.apache.mahout.vectorizer.collocations.llr.CollocDriver;
import org.apache.mahout.vectorizer.common.PartialVectorMerger;
@@ -152,19 +154,22 @@ public final class DictionaryVectorizer
List<Path> dictionaryChunks;
if (maxNGramSize == 1) {
startWordCounting(input, dictionaryJobPath, baseConf, minSupport);
- dictionaryChunks = createDictionaryChunks(dictionaryJobPath, output,
- baseConf, chunkSizeInMegabytes, new LongWritable(), maxTermDimension);
+ dictionaryChunks =
+ createDictionaryChunks(dictionaryJobPath, output, baseConf, chunkSizeInMegabytes, maxTermDimension);
} else {
CollocDriver.generateAllGrams(input, dictionaryJobPath, baseConf, maxNGramSize,
minSupport, minLLRValue, numReducers);
- dictionaryChunks = createDictionaryChunks(new Path(
- new Path(output, DICTIONARY_JOB_FOLDER), CollocDriver.NGRAM_OUTPUT_DIRECTORY), output,
- baseConf,
- chunkSizeInMegabytes, new DoubleWritable(), maxTermDimension);
+ dictionaryChunks =
+ createDictionaryChunks(new Path(new Path(output, DICTIONARY_JOB_FOLDER),
+ CollocDriver.NGRAM_OUTPUT_DIRECTORY),
+ output,
+ baseConf,
+ chunkSizeInMegabytes,
+ maxTermDimension);
}
int partialVectorIndex = 0;
- List<Path> partialVectorPaths = new ArrayList<Path>();
+ Collection<Path> partialVectorPaths = new ArrayList<Path>();
for (Path dictionaryChunk : dictionaryChunks) {
Path partialVectorOutputPath = new Path(output, VECTOR_OUTPUT_FOLDER + partialVectorIndex++);
partialVectorPaths.add(partialVectorOutputPath);
@@ -173,12 +178,11 @@ public final class DictionaryVectorizer
}
Configuration conf = new Configuration(baseConf);
- FileSystem fs = FileSystem.get(partialVectorPaths.get(0).toUri(), conf);
-
+
Path outputDir = new Path(output, DOCUMENT_VECTOR_OUTPUT_FOLDER);
PartialVectorMerger.mergePartialVectors(partialVectorPaths, outputDir, conf, normPower, logNormalize,
maxTermDimension[0], sequentialAccess, namedVectors, numReducers);
- HadoopUtil.deletePaths(partialVectorPaths, fs);
+ HadoopUtil.delete(conf, partialVectorPaths);
}
/**
@@ -189,48 +193,40 @@ public final class DictionaryVectorizer
Path dictionaryPathBase,
Configuration baseConf,
int chunkSizeInMegabytes,
- Writable value,
int[] maxTermDimension) throws IOException {
List<Path> chunkPaths = new ArrayList<Path>();
- Writable key = new Text();
Configuration conf = new Configuration(baseConf);
FileSystem fs = FileSystem.get(wordCountPath.toUri(), conf);
- FileStatus[] outputFiles = fs.globStatus(new Path(wordCountPath, OUTPUT_FILES_PATTERN));
-
+
long chunkSizeLimit = chunkSizeInMegabytes * 1024L * 1024L;
int chunkIndex = 0;
Path chunkPath = new Path(dictionaryPathBase, DICTIONARY_FILE + chunkIndex);
chunkPaths.add(chunkPath);
- SequenceFile.Writer dictWriter = new SequenceFile.Writer(fs, conf, chunkPath, Text.class,
- IntWritable.class);
+ SequenceFile.Writer dictWriter = new SequenceFile.Writer(fs, conf, chunkPath, Text.class, IntWritable.class);
long currentChunkSize = 0;
-
+ Path filesPattern = new Path(wordCountPath, OUTPUT_FILES_PATTERN);
int i = 0;
- for (FileStatus fileStatus : outputFiles) {
- Path path = fileStatus.getPath();
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
- // key is feature value is count
- while (reader.next(key, value)) {
- if (currentChunkSize > chunkSizeLimit) {
- dictWriter.close();
- chunkIndex++;
-
- chunkPath = new Path(dictionaryPathBase, DICTIONARY_FILE + chunkIndex);
- chunkPaths.add(chunkPath);
-
- dictWriter = new SequenceFile.Writer(fs, conf, chunkPath, Text.class, IntWritable.class);
- currentChunkSize = 0;
- }
-
- int fieldSize = DICTIONARY_BYTE_OVERHEAD + key.toString().length() * 2
- + Integer.SIZE / 8;
- currentChunkSize += fieldSize;
- dictWriter.append(key, new IntWritable(i++));
+ for (Pair<Writable,Writable> record :
+ new SequenceFileDirIterable<Writable,Writable>(filesPattern, PathType.GLOB, null, null, true, conf)) {
+ if (currentChunkSize > chunkSizeLimit) {
+ dictWriter.close();
+ chunkIndex++;
+
+ chunkPath = new Path(dictionaryPathBase, DICTIONARY_FILE + chunkIndex);
+ chunkPaths.add(chunkPath);
+
+ dictWriter = new SequenceFile.Writer(fs, conf, chunkPath, Text.class, IntWritable.class);
+ currentChunkSize = 0;
}
+
+ Writable key = record.getFirst();
+ int fieldSize = DICTIONARY_BYTE_OVERHEAD + key.toString().length() * 2 + Integer.SIZE / 8;
+ currentChunkSize += fieldSize;
+ dictWriter.append(key, new IntWritable(i++));
}
maxTermDimension[0] = i;
dictWriter.close();
@@ -300,7 +296,7 @@ public final class DictionaryVectorizer
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setNumReduceTasks(numReducers);
- HadoopUtil.overwriteOutput(output);
+ HadoopUtil.delete(conf, output);
job.waitForCompletion(true);
}
@@ -336,7 +332,7 @@ public final class DictionaryVectorizer
job.setReducerClass(TermCountReducer.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
- HadoopUtil.overwriteOutput(output);
+ HadoopUtil.delete(conf, output);
job.waitForCompletion(true);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DocumentProcessor.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DocumentProcessor.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DocumentProcessor.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DocumentProcessor.java Thu Mar 31 09:25:25 2011
@@ -89,7 +89,7 @@ public final class DocumentProcessor {
job.setInputFormatClass(SequenceFileInputFormat.class);
job.setNumReduceTasks(0);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
- HadoopUtil.overwriteOutput(output);
+ HadoopUtil.delete(conf, output);
job.waitForCompletion(true);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java Thu Mar 31 09:25:25 2011
@@ -52,6 +52,7 @@ public final class SparseVectorsFromSequ
ToolRunner.run(new SparseVectorsFromSequenceFiles(), args);
}
+ @Override
public int run(String[] args) throws Exception {
DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
ArgumentBuilder abuilder = new ArgumentBuilder();
@@ -172,7 +173,7 @@ public final class SparseVectorsFromSequ
log.info("Maximum n-gram size is: {}", maxNGramSize);
if (cmdLine.hasOption(overwriteOutput)) {
- HadoopUtil.overwriteOutput(outputDir);
+ HadoopUtil.delete(getConf(), outputDir);
}
float minLLRValue = LLRReducer.DEFAULT_MIN_LLR;
@@ -234,9 +235,9 @@ public final class SparseVectorsFromSequ
if (cmdLine.hasOption(logNormalizeOpt)) {
logNormalize = true;
}
-
- HadoopUtil.overwriteOutput(outputDir);
+
Configuration conf = getConf();
+ HadoopUtil.delete(conf, outputDir);
Path tokenizedPath = new Path(outputDir, DocumentProcessor.TOKENIZED_DOCUMENT_OUTPUT_FOLDER);
DocumentProcessor.tokenizeDocuments(inputDir, analyzerClass, tokenizedPath, conf);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java Thu Mar 31 09:25:25 2011
@@ -106,7 +106,7 @@ public final class CollocDriver extends
log.info("Maximum n-gram size is: {}", maxNGramSize);
if (argMap.containsKey("--overwrite")) {
- HadoopUtil.overwriteOutput(output);
+ HadoopUtil.delete(getConf(), output);
}
int minSupport = CollocReducer.DEFAULT_MIN_SUPPORT;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/common/PartialVectorMerger.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/common/PartialVectorMerger.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/common/PartialVectorMerger.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/common/PartialVectorMerger.java Thu Mar 31 09:25:25 2011
@@ -124,7 +124,7 @@ public final class PartialVectorMerger {
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setNumReduceTasks(numReducers);
- HadoopUtil.overwriteOutput(output);
+ HadoopUtil.delete(conf, output);
job.waitForCompletion(true);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java Thu Mar 31 09:25:25 2011
@@ -24,16 +24,16 @@ import java.util.Iterator;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.lucene.analysis.shingle.ShingleFilter;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.mahout.common.Pair;
import org.apache.mahout.common.StringTuple;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileIterable;
import org.apache.mahout.math.NamedVector;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.SequentialAccessSparseVector;
@@ -122,14 +122,10 @@ public class TFPartialVectorReducer exte
maxNGramSize = conf.getInt(DictionaryVectorizer.MAX_NGRAMS, maxNGramSize);
Path dictionaryFile = new Path(localFiles[0].getPath());
- FileSystem fs = dictionaryFile.getFileSystem(conf);
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, dictionaryFile, conf);
- Writable key = new Text();
- IntWritable value = new IntWritable();
-
// key is word value is id
- while (reader.next(key, value)) {
- dictionary.put(key.toString(), value.get());
+ for (Pair<Writable,IntWritable> record :
+ new SequenceFileIterable<Writable,IntWritable>(dictionaryFile, true, conf)) {
+ dictionary.put(record.getFirst().toString(), record.getSecond().get());
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFConverter.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFConverter.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFConverter.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFConverter.java Thu Mar 31 09:25:25 2011
@@ -25,7 +25,6 @@ import java.util.List;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
@@ -40,6 +39,8 @@ import org.apache.hadoop.mapreduce.lib.o
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.mahout.common.HadoopUtil;
import org.apache.mahout.common.Pair;
+import org.apache.mahout.common.iterator.sequencefile.PathType;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileDirIterable;
import org.apache.mahout.math.VectorWritable;
import org.apache.mahout.vectorizer.common.PartialVectorMerger;
import org.apache.mahout.vectorizer.term.TermDocumentCountMapper;
@@ -169,7 +170,6 @@ public final class TFIDFConverter {
}
Configuration conf = new Configuration(baseConf);
- FileSystem fs = FileSystem.get(partialVectorPaths.get(0).toUri(), conf);
Path outputDir = new Path(output, DOCUMENT_VECTOR_OUTPUT_FOLDER);
@@ -182,7 +182,7 @@ public final class TFIDFConverter {
sequentialAccessOutput,
namedVector,
numReducers);
- HadoopUtil.deletePaths(partialVectorPaths, fs);
+ HadoopUtil.delete(conf, partialVectorPaths);
}
@@ -195,13 +195,9 @@ public final class TFIDFConverter {
Configuration baseConf,
int chunkSizeInMegabytes) throws IOException {
List<Path> chunkPaths = new ArrayList<Path>();
-
- IntWritable key = new IntWritable();
- LongWritable value = new LongWritable();
Configuration conf = new Configuration(baseConf);
FileSystem fs = FileSystem.get(featureCountPath.toUri(), conf);
- FileStatus[] outputFiles = fs.globStatus(new Path(featureCountPath, OUTPUT_FILES_PATTERN));
long chunkSizeLimit = chunkSizeInMegabytes * 1024L * 1024L;
int chunkIndex = 0;
@@ -213,34 +209,34 @@ public final class TFIDFConverter {
long currentChunkSize = 0;
long featureCount = 0;
long vectorCount = Long.MAX_VALUE;
- for (FileStatus fileStatus : outputFiles) {
- Path path = fileStatus.getPath();
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
- // key is feature value is count
- while (reader.next(key, value)) {
- if (currentChunkSize > chunkSizeLimit) {
- freqWriter.close();
- chunkIndex++;
-
- chunkPath = new Path(dictionaryPathBase, FREQUENCY_FILE + chunkIndex);
- chunkPaths.add(chunkPath);
-
- freqWriter = new SequenceFile.Writer(fs, conf, chunkPath, IntWritable.class, LongWritable.class);
- currentChunkSize = 0;
- }
-
- int fieldSize = SEQUENCEFILE_BYTE_OVERHEAD + Integer.SIZE / 8 + Long.SIZE / 8;
- currentChunkSize += fieldSize;
- if (key.get() >= 0) {
- freqWriter.append(key, value);
- } else if (key.get() == -1) {
- vectorCount = value.get();
- }
- featureCount = Math.max(key.get(), featureCount);
+ Path filesPattern = new Path(featureCountPath, OUTPUT_FILES_PATTERN);
+ for (Pair<IntWritable,LongWritable> record :
+ new SequenceFileDirIterable<IntWritable,LongWritable>(filesPattern, PathType.GLOB, null, null, true, conf)) {
+
+ if (currentChunkSize > chunkSizeLimit) {
+ freqWriter.close();
+ chunkIndex++;
+
+ chunkPath = new Path(dictionaryPathBase, FREQUENCY_FILE + chunkIndex);
+ chunkPaths.add(chunkPath);
+ freqWriter = new SequenceFile.Writer(fs, conf, chunkPath, IntWritable.class, LongWritable.class);
+ currentChunkSize = 0;
}
+
+ int fieldSize = SEQUENCEFILE_BYTE_OVERHEAD + Integer.SIZE / 8 + Long.SIZE / 8;
+ currentChunkSize += fieldSize;
+ IntWritable key = record.getFirst();
+ LongWritable value = record.getSecond();
+ if (key.get() >= 0) {
+ freqWriter.append(key, value);
+ } else if (key.get() == -1) {
+ vectorCount = value.get();
+ }
+ featureCount = Math.max(key.get(), featureCount);
+
}
- featureCount++;
+ featureCount++;
freqWriter.close();
Long[] counts = {featureCount, vectorCount};
return new Pair<Long[], List<Path>>(counts, chunkPaths);
@@ -309,7 +305,7 @@ public final class TFIDFConverter {
job.setReducerClass(TFIDFPartialVectorReducer.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
- HadoopUtil.overwriteOutput(output);
+ HadoopUtil.delete(conf, output);
job.waitForCompletion(true);
}
@@ -343,7 +339,7 @@ public final class TFIDFConverter {
job.setReducerClass(TermDocumentCountReducer.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
- HadoopUtil.overwriteOutput(output);
+ HadoopUtil.delete(conf, output);
job.waitForCompletion(true);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFPartialVectorReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFPartialVectorReducer.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFPartialVectorReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFPartialVectorReducer.java Thu Mar 31 09:25:25 2011
@@ -24,13 +24,13 @@ import java.util.Iterator;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.mahout.common.Pair;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileIterable;
import org.apache.mahout.math.NamedVector;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.SequentialAccessSparseVector;
@@ -114,14 +114,10 @@ public class TFIDFPartialVectorReducer e
namedVector = conf.getBoolean(PartialVectorMerger.NAMED_VECTOR, false);
Path dictionaryFile = new Path(localFiles[0].getPath());
- FileSystem fs = dictionaryFile.getFileSystem(conf);
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, dictionaryFile, conf);
- IntWritable key = new IntWritable();
- LongWritable value = new LongWritable();
-
// key is feature, value is the document frequency
- while (reader.next(key, value)) {
- dictionary.put(key.get(), value.get());
+ for (Pair<IntWritable,LongWritable> record :
+ new SequenceFileIterable<IntWritable,LongWritable>(dictionaryFile, true, conf)) {
+ dictionary.put(record.getFirst().get(), record.getSecond().get());
}
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/als/ParallelALSFactorizationJobTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/als/ParallelALSFactorizationJobTest.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/als/ParallelALSFactorizationJobTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/als/ParallelALSFactorizationJobTest.java Thu Mar 31 09:25:25 2011
@@ -18,7 +18,6 @@
package org.apache.mahout.cf.taste.hadoop.als;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.LongWritable;
@@ -28,6 +27,7 @@ import org.apache.hadoop.mapreduce.Reduc
import org.apache.mahout.cf.taste.hadoop.TasteHadoopUtils;
import org.apache.mahout.cf.taste.impl.TasteTestCase;
import org.apache.mahout.cf.taste.impl.common.FullRunningAverage;
+import org.apache.mahout.cf.taste.impl.common.RunningAverage;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.Matrix;
import org.apache.mahout.math.MatrixSlice;
@@ -81,12 +81,12 @@ public class ParallelALSFactorizationJob
public void initializeMReducer() throws Exception {
Reducer<VarLongWritable,FloatWritable,VarIntWritable,FeatureVectorWithRatingWritable>.Context ctx =
EasyMock.createMock(Reducer.Context.class);
- ctx.write(EasyMock.eq(new VarIntWritable(TasteHadoopUtils.idToIndex(123L))), matchInitializedFeatureVector(3d, 3));
+ ctx.write(EasyMock.eq(new VarIntWritable(TasteHadoopUtils.idToIndex(123L))), matchInitializedFeatureVector(3.0, 3));
EasyMock.replay(ctx);
ParallelALSFactorizationJob.InitializeMReducer reducer = new ParallelALSFactorizationJob.InitializeMReducer();
setField(reducer, "numFeatures", 3);
- reducer.reduce(new VarLongWritable(123L), Arrays.asList(new FloatWritable(4f), new FloatWritable(2f)), ctx);
+ reducer.reduce(new VarLongWritable(123L), Arrays.asList(new FloatWritable(4.0f), new FloatWritable(2.0f)), ctx);
EasyMock.verify(ctx);
}
@@ -215,10 +215,10 @@ public class ParallelALSFactorizationJob
Double na = Double.NaN;
Matrix preferences = new SparseRowMatrix(new int[] { 4, 4 }, new Vector[] {
- new DenseVector(new double[] { 5d, 5d, 2d, na }),
- new DenseVector(new double[] { 2d, na, 3d, 5d }),
- new DenseVector(new double[] { na, 5d, na, 3d }),
- new DenseVector(new double[] { 3d, na, na, 5d }) });
+ new DenseVector(new double[] {5.0, 5.0, 2.0, na }),
+ new DenseVector(new double[] {2.0, na, 3.0, 5.0 }),
+ new DenseVector(new double[] { na, 5.0, na, 3.0 }),
+ new DenseVector(new double[] {3.0, na, na, 5.0 }) });
StringBuilder prefsAsText = new StringBuilder();
String separator = "";
@@ -229,7 +229,7 @@ public class ParallelALSFactorizationJob
while (elementIterator.hasNext()) {
Vector.Element e = elementIterator.next();
if (!Double.isNaN(e.get())) {
- prefsAsText.append(separator).append(slice.index()).append(",").append(e.index()).append(",").append(e.get());
+ prefsAsText.append(separator).append(slice.index()).append(',').append(e.index()).append(',').append(e.get());
separator = "\n";
}
}
@@ -244,23 +244,19 @@ public class ParallelALSFactorizationJob
conf.set("mapred.output.dir", outputDir.getAbsolutePath());
conf.setBoolean("mapred.output.compress", false);
+ alsFactorization.setConf(conf);
int numFeatures = 3;
int numIterations = 5;
double lambda = 0.065;
-
- alsFactorization.setConf(conf);
alsFactorization.run(new String[] { "--tempDir", tmpDir.getAbsolutePath(), "--lambda", String.valueOf(lambda),
"--numFeatures", String.valueOf(numFeatures), "--numIterations", String.valueOf(numIterations) });
- Path inputPath = new Path(inputFile.getAbsolutePath());
- FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
-
- Matrix u = MathHelper.readEntries(fs, conf, new Path(outputDir.getAbsolutePath(), "U/part-r-00000"),
+ Matrix u = MathHelper.readEntries(conf, new Path(outputDir.getAbsolutePath(), "U/part-r-00000"),
preferences.numRows(), numFeatures);
- Matrix m = MathHelper.readEntries(fs, conf, new Path(outputDir.getAbsolutePath(), "M/part-r-00000"),
+ Matrix m = MathHelper.readEntries(conf, new Path(outputDir.getAbsolutePath(), "M/part-r-00000"),
preferences.numCols(), numFeatures);
- FullRunningAverage avg = new FullRunningAverage();
+ RunningAverage avg = new FullRunningAverage();
sliceIterator = preferences.iterateAll();
while (sliceIterator.hasNext()) {
MatrixSlice slice = sliceIterator.next();
@@ -273,7 +269,7 @@ public class ParallelALSFactorizationJob
double err = pref - estimate;
avg.addDatum(err * err);
logger.info("Comparing preference of user [" + slice.index() + "] towards item [" + e.index() + "], " +
- "was [" + pref + "] estimate is [" + estimate + "]");
+ "was [" + pref + "] estimate is [" + estimate + ']');
}
}
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java Thu Mar 31 09:25:25 2011
@@ -41,7 +41,7 @@ import org.apache.mahout.cf.taste.impl.T
import org.apache.mahout.cf.taste.impl.common.FastIDSet;
import org.apache.mahout.cf.taste.impl.recommender.GenericRecommendedItem;
import org.apache.mahout.cf.taste.recommender.RecommendedItem;
-import org.apache.mahout.common.FileLineIterable;
+import org.apache.mahout.common.iterator.FileLineIterable;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.VarIntWritable;
import org.apache.mahout.math.VarLongWritable;
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java Thu Mar 31 09:25:25 2011
@@ -107,8 +107,7 @@ public final class BayesClassifierSelfTe
TestClassifier.classifyParallel(params);
Configuration conf = new Configuration();
Path outputFiles = getTestTempFilePath("bayesinput-output/part*");
- FileSystem fs = FileSystem.get(outputFiles.toUri(), conf);
- matrix = BayesClassifierDriver.readResult(fs, outputFiles, conf, params).getConfusionMatrix();
+ matrix = BayesClassifierDriver.readResult(outputFiles, conf, params).getConfusionMatrix();
for (int i = 0; i < 3; i++) {
for (int j = 0; j < 3; j++) {
assertEquals(i == j ? 4 : 0, matrix[i][j]);
@@ -158,8 +157,7 @@ public final class BayesClassifierSelfTe
TestClassifier.classifyParallel(params);
Configuration conf = new Configuration();
Path outputFiles = getTestTempFilePath("bayesinput-output/part*");
- FileSystem fs = FileSystem.get(outputFiles.toUri(), conf);
- matrix = BayesClassifierDriver.readResult(fs, outputFiles, conf, params).getConfusionMatrix();
+ matrix = BayesClassifierDriver.readResult(outputFiles, conf, params).getConfusionMatrix();
for (int i = 0; i < 3; i++) {
for (int j = 0; j < 3; j++) {
assertEquals(i == j ? 4 : 0, matrix[i][j]);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java Thu Mar 31 09:25:25 2011
@@ -27,8 +27,8 @@ import java.util.Iterator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.mahout.classifier.BayesFileFormatter;
-import org.apache.mahout.common.FileLineIterable;
-import org.apache.mahout.common.FileLineIterator;
+import org.apache.mahout.common.iterator.FileLineIterable;
+import org.apache.mahout.common.iterator.FileLineIterator;
import org.apache.mahout.common.MahoutTestCase;
import org.junit.Before;
import org.junit.Test;
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java Thu Mar 31 09:25:25 2011
@@ -35,16 +35,17 @@ import org.apache.hadoop.io.WritableComp
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.mahout.clustering.AbstractCluster;
import org.apache.mahout.clustering.ClusteringTestUtils;
import org.apache.mahout.clustering.WeightedVectorWritable;
import org.apache.mahout.common.DummyRecordWriter;
+import org.apache.mahout.common.HadoopUtil;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.commandline.DefaultOptionCreator;
import org.apache.mahout.common.distance.DistanceMeasure;
import org.apache.mahout.common.distance.EuclideanDistanceMeasure;
import org.apache.mahout.common.distance.ManhattanDistanceMeasure;
import org.apache.mahout.common.distance.UserDefinedDistanceMeasure;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileValueIterable;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
@@ -435,27 +436,15 @@ public final class TestCanopyCreation ex
// verify output from sequence file
Path path = new Path(output, "clusters-0/part-r-00000");
- FileSystem fs = FileSystem.get(path.toUri(), config);
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, config);
- Writable key = new Text();
- Canopy canopy = new Canopy();
int ix = 0;
- while (reader.next(key, canopy)) {
- assertEquals("Center [" + ix + ']', manhattanCentroids.get(ix), canopy.getCenter());
+ for (Canopy value : new SequenceFileValueIterable<Canopy>(path, true, config)) {
+ assertEquals("Center [" + ix + ']', manhattanCentroids.get(ix), value.getCenter());
ix++;
}
- reader.close();
+
path = new Path(output, "clusteredPoints/part-m-0");
- reader = new SequenceFile.Reader(fs, path, config);
- int count = 0;
- Writable clusterId = new IntWritable(0);
- WeightedVectorWritable vector = new WeightedVectorWritable();
- while (reader.next(clusterId, vector)) {
- count++;
- System.out.println("Txt: " + clusterId + " Vec: " + AbstractCluster.formatVector(vector.getVector(), null));
- }
+ long count = HadoopUtil.countRecords(path, config);
assertEquals("number of points", points.size(), count);
- reader.close();
}
/** Story: User can cluster points using sequential execution */
@@ -476,27 +465,16 @@ public final class TestCanopyCreation ex
// verify output from sequence file
Path path = new Path(output, "clusters-0/part-r-00000");
- FileSystem fs = FileSystem.get(path.toUri(), config);
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, config);
- Writable key = new Text();
- Canopy canopy = new Canopy();
+
int ix = 0;
- while (reader.next(key, canopy)) {
- assertEquals("Center [" + ix + ']', euclideanCentroids.get(ix), canopy.getCenter());
+ for (Canopy value : new SequenceFileValueIterable<Canopy>(path, true, config)) {
+ assertEquals("Center [" + ix + ']', euclideanCentroids.get(ix), value.getCenter());
ix++;
}
- reader.close();
+
path = new Path(output, "clusteredPoints/part-m-0");
- reader = new SequenceFile.Reader(fs, path, config);
- int count = 0;
- Writable clusterId = new IntWritable(0);
- WeightedVectorWritable vector = new WeightedVectorWritable();
- while (reader.next(clusterId, vector)) {
- count++;
- System.out.println("Txt: " + clusterId + " Vec: " + AbstractCluster.formatVector(vector.getVector(), null));
- }
+ long count = HadoopUtil.countRecords(path, config);
assertEquals("number of points", points.size(), count);
- reader.close();
}
/**
@@ -513,16 +491,8 @@ public final class TestCanopyCreation ex
Path output = getTestTempDirPath("output");
CanopyDriver.run(conf, getTestTempDirPath("testdata"), output, manhattanDistanceMeasure, 3.1, 2.1, true, false);
Path path = new Path(output, "clusteredPoints/part-m-00000");
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
- int count = 0;
- Writable clusterId = new IntWritable(0);
- WeightedVectorWritable vector = new WeightedVectorWritable();
- while (reader.next(clusterId, vector)) {
- count++;
- System.out.println("Txt: " + clusterId + " Vec: " + AbstractCluster.formatVector(vector.getVector(), null));
- }
+ long count = HadoopUtil.countRecords(path, conf);
assertEquals("number of points", points.size(), count);
- reader.close();
}
/**
@@ -544,16 +514,8 @@ public final class TestCanopyCreation ex
optKey(DefaultOptionCreator.OVERWRITE_OPTION) };
ToolRunner.run(new Configuration(), new CanopyDriver(), args);
Path path = new Path(output, "clusteredPoints/part-m-00000");
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
- int count = 0;
- Writable canopyId = new IntWritable(0);
- WeightedVectorWritable vw = new WeightedVectorWritable();
- while (reader.next(canopyId, vw)) {
- count++;
- System.out.println("Txt: " + canopyId.toString() + " Vec: " + AbstractCluster.formatVector(vw.getVector(), null));
- }
+ long count = HadoopUtil.countRecords(path, conf);
assertEquals("number of points", points.size(), count);
- reader.close();
}
/** Story: Clustering algorithm must support arbitrary user defined distance measure */
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java Thu Mar 31 09:25:25 2011
@@ -31,7 +31,6 @@ import org.apache.hadoop.io.DataOutputBu
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
@@ -42,6 +41,7 @@ import org.apache.mahout.clustering.Clus
import org.apache.mahout.clustering.WeightedVectorWritable;
import org.apache.mahout.clustering.kmeans.TestKmeansClustering;
import org.apache.mahout.common.DummyRecordWriter;
+import org.apache.mahout.common.HadoopUtil;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.commandline.DefaultOptionCreator;
import org.apache.mahout.common.distance.DistanceMeasure;
@@ -242,13 +242,8 @@ public final class TestFuzzyKmeansCluste
DefaultOptionCreator.SEQUENTIAL_METHOD
};
new FuzzyKMeansDriver().run(args);
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(output, "clusteredPoints/part-m-0"), conf);
- Writable key = new IntWritable();
- Writable out = new WeightedVectorWritable();
- while (reader.next(key, out)) {
- // make sure we can read all the clusters
- }
- reader.close();
+ long count = HadoopUtil.countRecords(new Path(output, "clusteredPoints/part-m-0"), conf);
+ assertTrue(count > 0);
}
}
@@ -318,13 +313,8 @@ public final class TestFuzzyKmeansCluste
optKey(DefaultOptionCreator.OVERWRITE_OPTION)
};
ToolRunner.run(new Configuration(), new FuzzyKMeansDriver(), args);
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(output, "clusteredPoints/part-m-00000"), conf);
- Writable key = new IntWritable();
- Writable out = new WeightedVectorWritable();
- while (reader.next(key, out)) {
- // make sure we can read all the clusters
- }
- reader.close();
+ long count = HadoopUtil.countRecords(new Path(output, "clusteredPoints/part-m-00000"), conf);
+ assertTrue(count > 0);
}
}
@@ -612,7 +602,8 @@ public final class TestFuzzyKmeansCluste
}
FuzzyKMeansClusterMapper clusterMapper = new FuzzyKMeansClusterMapper();
- DummyRecordWriter<IntWritable, WeightedVectorWritable> clusterWriter = new DummyRecordWriter<IntWritable, WeightedVectorWritable>();
+ DummyRecordWriter<IntWritable, WeightedVectorWritable> clusterWriter =
+ new DummyRecordWriter<IntWritable, WeightedVectorWritable>();
Mapper<WritableComparable<?>, VectorWritable, IntWritable, WeightedVectorWritable>.Context clusterContext =
DummyRecordWriter.build(clusterMapper, conf, clusterWriter);
clusterMapper.setup(reducerClusters, conf);
@@ -640,9 +631,8 @@ public final class TestFuzzyKmeansCluste
2,
1);
- computeCluster(pointsVectors, clusters.get(clusters.size() - 1), new FuzzyKMeansClusterer(new EuclideanDistanceMeasure(),
- 0.001,
- 2), refClusters);
+ computeCluster(pointsVectors, clusters.get(clusters.size() - 1),
+ new FuzzyKMeansClusterer(new EuclideanDistanceMeasure(), 0.001, 2), refClusters);
// Now compare the clustermapper results with reference implementation
assertEquals("mapper and reference sizes", refClusters.size(), clusterWriter.getKeys().size());
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java Thu Mar 31 09:25:25 2011
@@ -42,10 +42,12 @@ import org.apache.mahout.clustering.cano
import org.apache.mahout.common.DummyOutputCollector;
import org.apache.mahout.common.DummyRecordWriter;
import org.apache.mahout.common.MahoutTestCase;
+import org.apache.mahout.common.Pair;
import org.apache.mahout.common.commandline.DefaultOptionCreator;
import org.apache.mahout.common.distance.DistanceMeasure;
import org.apache.mahout.common.distance.EuclideanDistanceMeasure;
import org.apache.mahout.common.distance.ManhattanDistanceMeasure;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileIterable;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.SequentialAccessSparseVector;
@@ -395,20 +397,15 @@ public final class TestKmeansClustering
// now compare the expected clusters with actual
Path clusteredPointsPath = new Path(outputPath, "clusteredPoints");
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(clusteredPointsPath, "part-m-0"), conf);
int[] expect = EXPECTED_NUM_POINTS[k];
DummyOutputCollector<IntWritable, WeightedVectorWritable> collector =
new DummyOutputCollector<IntWritable, WeightedVectorWritable>();
- // The key is the clusterId
- IntWritable clusterId = new IntWritable(0);
- // The value is the weighted vector
- WeightedVectorWritable value = new WeightedVectorWritable();
- while (reader.next(clusterId, value)) {
- collector.collect(clusterId, value);
- clusterId = new IntWritable(0);
- value = new WeightedVectorWritable();
+ // The key is the clusterId, the value is the weighted vector
+ for (Pair<IntWritable,WeightedVectorWritable> record :
+ new SequenceFileIterable<IntWritable,WeightedVectorWritable>(
+ new Path(clusteredPointsPath, "part-m-0"), conf)) {
+ collector.collect(record.getFirst(), record.getSecond());
}
- reader.close();
assertEquals("clusters[" + k + ']', expect.length, collector.getKeys().size());
}
}
@@ -453,20 +450,15 @@ public final class TestKmeansClustering
// now compare the expected clusters with actual
Path clusteredPointsPath = new Path(outputPath, "clusteredPoints");
// assertEquals("output dir files?", 4, outFiles.length);
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(clusteredPointsPath, "part-m-00000"), conf);
int[] expect = EXPECTED_NUM_POINTS[k];
DummyOutputCollector<IntWritable, WeightedVectorWritable> collector =
new DummyOutputCollector<IntWritable, WeightedVectorWritable>();
- // The key is the clusterId
- IntWritable clusterId = new IntWritable(0);
- // The value is the weighted vector
- WeightedVectorWritable value = new WeightedVectorWritable();
- while (reader.next(clusterId, value)) {
- collector.collect(clusterId, value);
- clusterId = new IntWritable(0);
- value = new WeightedVectorWritable();
+ // The key is the clusterId, the value is the weighted vector
+ for (Pair<IntWritable,WeightedVectorWritable> record :
+ new SequenceFileIterable<IntWritable,WeightedVectorWritable>(
+ new Path(clusteredPointsPath, "part-m-00000"), conf)) {
+ collector.collect(record.getFirst(), record.getSecond());
}
- reader.close();
if (k == 2) {
// cluster 3 is empty so won't appear in output
assertEquals("clusters[" + k + ']', expect.length - 1, collector.getKeys().size());
@@ -504,19 +496,13 @@ public final class TestKmeansClustering
Path clusteredPointsPath = new Path(outputPath, "clusteredPoints");
DummyOutputCollector<IntWritable, WeightedVectorWritable> collector =
new DummyOutputCollector<IntWritable, WeightedVectorWritable>();
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(clusteredPointsPath, "part-m-00000"), conf);
-
- // The key is the clusterId
- IntWritable clusterId = new IntWritable(0);
- // The value is the vector
- WeightedVectorWritable value = new WeightedVectorWritable();
- while (reader.next(clusterId, value)) {
- collector.collect(clusterId, value);
- clusterId = new IntWritable(0);
- value = new WeightedVectorWritable();
+ // The key is the clusterId, the value is the weighted vector
+ for (Pair<IntWritable,WeightedVectorWritable> record :
+ new SequenceFileIterable<IntWritable,WeightedVectorWritable>(
+ new Path(clusteredPointsPath, "part-m-00000"), conf)) {
+ collector.collect(record.getFirst(), record.getSecond());
}
- reader.close();
assertEquals("num points[0]", 4, collector.getValue(new IntWritable(0)).size());
assertEquals("num points[1]", 5, collector.getValue(new IntWritable(1)).size());
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestRandomSeedGenerator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestRandomSeedGenerator.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestRandomSeedGenerator.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestRandomSeedGenerator.java Thu Mar 31 09:25:25 2011
@@ -25,13 +25,12 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.mahout.clustering.AbstractCluster;
import org.apache.mahout.clustering.ClusteringTestUtils;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.distance.ManhattanDistanceMeasure;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileValueIterable;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
@@ -75,14 +74,11 @@ public final class TestRandomSeedGenerat
ClusteringTestUtils.writePointsToFile(points, input, fs, conf);
RandomSeedGenerator.buildRandom(input, output, 4, new ManhattanDistanceMeasure());
-
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(output, "part-randomSeed"), conf);
- Writable key = reader.getKeyClass().asSubclass(Writable.class).newInstance();
- AbstractCluster value = (AbstractCluster) reader.getValueClass().newInstance();
-
+
int clusterCount = 0;
Collection<Integer> set = new HashSet<Integer>();
- while (reader.next(key, value)) {
+ for (AbstractCluster value :
+ new SequenceFileValueIterable<AbstractCluster>(new Path(output, "part-randomSeed"), true, conf)) {
clusterCount++;
int id = value.getId();
assertTrue(set.add(id)); // validate unique id's
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java Thu Mar 31 09:25:25 2011
@@ -27,15 +27,14 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.util.ToolRunner;
import org.apache.mahout.clustering.ClusteringTestUtils;
import org.apache.mahout.common.DummyRecordWriter;
+import org.apache.mahout.common.HadoopUtil;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.commandline.DefaultOptionCreator;
import org.apache.mahout.common.distance.DistanceMeasure;
@@ -151,7 +150,8 @@ public final class TestMeanShift extends
@Test
public void testClustererReferenceImplementation() {
Iterable<Vector> points = new ArrayList<Vector>(Arrays.asList(raw));
- List<MeanShiftCanopy> canopies = MeanShiftCanopyClusterer.clusterPoints(points, euclideanDistanceMeasure, 0.5, 4, 1, 10);
+ List<MeanShiftCanopy> canopies =
+ MeanShiftCanopyClusterer.clusterPoints(points, euclideanDistanceMeasure, 0.5, 4, 1, 10);
printCanopies(canopies);
printImage(canopies);
}
@@ -181,9 +181,8 @@ public final class TestMeanShift extends
// map the data
MeanShiftCanopyMapper mapper = new MeanShiftCanopyMapper();
DummyRecordWriter<Text, MeanShiftCanopy> mapWriter = new DummyRecordWriter<Text, MeanShiftCanopy>();
- Mapper<WritableComparable<?>, MeanShiftCanopy, Text, MeanShiftCanopy>.Context mapContext = DummyRecordWriter.build(mapper,
- conf,
- mapWriter);
+ Mapper<WritableComparable<?>, MeanShiftCanopy, Text, MeanShiftCanopy>.Context mapContext =
+ DummyRecordWriter.build(mapper, conf, mapWriter);
mapper.setup(mapContext);
for (MeanShiftCanopy canopy : canopies) {
mapper.map(new Text(), canopy, mapContext);
@@ -253,9 +252,8 @@ public final class TestMeanShift extends
MeanShiftCanopyMapper mapper = new MeanShiftCanopyMapper();
DummyRecordWriter<Text, MeanShiftCanopy> mapWriter = new DummyRecordWriter<Text, MeanShiftCanopy>();
- Mapper<WritableComparable<?>, MeanShiftCanopy, Text, MeanShiftCanopy>.Context mapContext = DummyRecordWriter.build(mapper,
- conf,
- mapWriter);
+ Mapper<WritableComparable<?>, MeanShiftCanopy, Text, MeanShiftCanopy>.Context mapContext =
+ DummyRecordWriter.build(mapper, conf, mapWriter);
mapper.setup(mapContext);
// map the data
@@ -268,11 +266,8 @@ public final class TestMeanShift extends
// now reduce the mapper output
MeanShiftCanopyReducer reducer = new MeanShiftCanopyReducer();
DummyRecordWriter<Text, MeanShiftCanopy> reduceWriter = new DummyRecordWriter<Text, MeanShiftCanopy>();
- Reducer<Text, MeanShiftCanopy, Text, MeanShiftCanopy>.Context reduceContext = DummyRecordWriter.build(reducer,
- conf,
- reduceWriter,
- Text.class,
- MeanShiftCanopy.class);
+ Reducer<Text, MeanShiftCanopy, Text, MeanShiftCanopy>.Context reduceContext =
+ DummyRecordWriter.build(reducer, conf, reduceWriter, Text.class, MeanShiftCanopy.class);
reducer.setup(reduceContext);
reducer.reduce(new Text("canopy"), mapWriter.getValue(new Text("canopy")), reduceContext);
reducer.cleanup(reduceContext);
@@ -330,14 +325,7 @@ public final class TestMeanShift extends
optKey(DefaultOptionCreator.OVERWRITE_OPTION) };
ToolRunner.run(conf, new MeanShiftCanopyDriver(), args);
Path outPart = new Path(output, "clusters-3/part-r-00000");
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, outPart, conf);
- Writable key = new Text();
- Writable value = new MeanShiftCanopy();
- int count = 0;
- while (reader.next(key, value)) {
- count++;
- }
- reader.close();
+ long count = HadoopUtil.countRecords(outPart, conf);
assertEquals("count", 3, count);
}
@@ -369,14 +357,7 @@ public final class TestMeanShift extends
DefaultOptionCreator.SEQUENTIAL_METHOD };
ToolRunner.run(new Configuration(), new MeanShiftCanopyDriver(), args);
Path outPart = new Path(output, "clusters-4/part-r-00000");
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, outPart, conf);
- Writable key = new Text();
- Writable value = new MeanShiftCanopy();
- int count = 0;
- while (reader.next(key, value)) {
- count++;
- }
- reader.close();
+ long count = HadoopUtil.countRecords(outPart, conf);
assertEquals("count", 5, count);
}
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java Thu Mar 31 09:25:25 2011
@@ -25,8 +25,10 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.ToolRunner;
import org.apache.mahout.clustering.minhash.HashFactory.HashType;
import org.apache.mahout.common.MahoutTestCase;
+import org.apache.mahout.common.Pair;
import org.apache.mahout.common.commandline.DefaultOptionCreator;
import org.apache.mahout.common.commandline.MinhashOptionCreator;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileIterable;
import org.apache.mahout.math.SequentialAccessSparseVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
@@ -45,8 +47,7 @@ public class TestMinHashClustering exten
{4, 7, 8, 9, 6, 1}, {5, 8, 10, 4, 1}, {6, 17, 14, 15},
{8, 9, 11, 6, 12, 1, 7}, {10, 13, 9, 7, 4, 6, 3},
{3, 5, 7, 9, 2, 11}, {13, 7, 6, 8, 5}};
-
- private FileSystem fs;
+
private Path input;
private Path output;
@@ -64,13 +65,12 @@ public class TestMinHashClustering exten
public void setUp() throws Exception {
super.setUp();
Configuration conf = new Configuration();
- fs = FileSystem.get(conf);
+ FileSystem fs = FileSystem.get(conf);
List<VectorWritable> points = getPointsWritable(REFERENCE);
input = getTestTempDirPath("points");
output = new Path(getTestTempDirPath(), "output");
Path pointFile = new Path(input, "file1");
- SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, pointFile, Text.class,
- VectorWritable.class);
+ SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, pointFile, Text.class, VectorWritable.class);
int id = 0;
for (VectorWritable point : points) {
writer.append(new Text("Id-" + id++), point);
@@ -123,22 +123,21 @@ public class TestMinHashClustering exten
}
}
- private void verify(Path output, double simThreshold, String msg) throws Exception {
+ private static void verify(Path output, double simThreshold, String msg) {
Configuration conf = new Configuration();
Path outputFile = new Path(output, "part-r-00000");
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, outputFile, conf);
- Writable clusterId = new Text();
- VectorWritable point = new VectorWritable();
List<Vector> clusteredItems = new ArrayList<Vector>();
String prevClusterId = "";
- while (reader.next(clusterId, point)) {
+ for (Pair<Writable,VectorWritable> record : new SequenceFileIterable<Writable,VectorWritable>(outputFile, conf)) {
+ Writable clusterId = record.getFirst();
+ VectorWritable point = record.getSecond();
if (prevClusterId.equals(clusterId.toString())) {
- clusteredItems.add(point.get().clone());
+ clusteredItems.add(point.get());
} else {
runPairwiseSimilarity(clusteredItems, simThreshold, msg);
clusteredItems.clear();
prevClusterId = clusterId.toString();
- clusteredItems.add(point.get().clone());
+ clusteredItems.add(point.get());
}
}
runPairwiseSimilarity(clusteredItems, simThreshold, msg);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/spectral/common/TestVectorCache.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/spectral/common/TestVectorCache.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/spectral/common/TestVectorCache.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/spectral/common/TestVectorCache.java Thu Mar 31 09:25:25 2011
@@ -28,6 +28,7 @@ import org.apache.hadoop.io.SequenceFile
import org.apache.hadoop.io.Writable;
import org.apache.mahout.common.HadoopUtil;
import org.apache.mahout.common.MahoutTestCase;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileValueIterator;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
@@ -40,7 +41,6 @@ public class TestVectorCache extends Mah
@Test
public void testSave() throws Exception {
Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
Writable key = new IntWritable(0);
Vector value = new DenseVector(VECTOR);
Path path = getTestTempDirPath("output");
@@ -49,10 +49,10 @@ public class TestVectorCache extends Mah
VectorCache.save(key, value, path, conf, true, true);
// can we read it from here?
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
- VectorWritable old = new VectorWritable();
- reader.next(key, old);
- reader.close();
+ SequenceFileValueIterator<VectorWritable> iterator =
+ new SequenceFileValueIterator<VectorWritable>(path, true, conf);
+ VectorWritable old = iterator.next();
+ iterator.close();
// test if the values are identical
assertEquals("Saved vector is identical to original", old.get(), value);
@@ -70,15 +70,14 @@ public class TestVectorCache extends Mah
// write the vector
path = fs.makeQualified(path);
fs.deleteOnExit(path);
- HadoopUtil.overwriteOutput(path);
+ HadoopUtil.delete(conf, path);
DistributedCache.setCacheFiles(new URI[] {path.toUri()}, conf);
- SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path,
- IntWritable.class, VectorWritable.class);
+ SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class);
writer.append(key, new VectorWritable(value));
writer.close();
// load it
- Vector result = VectorCache.load(key, conf);
+ Vector result = VectorCache.load(conf);
// are they the same?
assertNotNull("Vector is not null", result);
@@ -90,14 +89,13 @@ public class TestVectorCache extends Mah
Configuration conf = new Configuration();
Vector v = new DenseVector(VECTOR);
Path toSave = getTestTempDirPath("output");
- IntWritable key = new IntWritable(0);
+ Writable key = new IntWritable(0);
// save it
VectorCache.save(key, v, toSave, conf);
// now, load it back
- key = new IntWritable(0);
- Vector v2 = VectorCache.load(key, conf);
+ Vector v2 = VectorCache.load(conf);
// are they the same?
assertNotNull("Vector is not null", v2);