You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2011/03/31 11:25:30 UTC

svn commit: r1087225 [5/7] - in /mahout/trunk: core/src/main/java/org/apache/mahout/cf/taste/hadoop/ core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/ core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/ core/src/main/java/org/apache/m...

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDPrototype.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDPrototype.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDPrototype.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDPrototype.java Thu Mar 31 09:25:25 2011
@@ -20,11 +20,12 @@ package org.apache.mahout.math.hadoop.st
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
+import org.apache.mahout.common.RandomUtils;
+import org.apache.mahout.common.iterator.CopyConstructorIterator;
 import org.apache.mahout.math.DenseMatrix;
 import org.apache.mahout.math.DenseVector;
 import org.apache.mahout.math.SingularValueDecomposition;
@@ -34,26 +35,21 @@ import org.apache.mahout.math.ssvd.Eigen
 /**
  * SSVD protoptype: non-MR concept verification for Givens QR & SSVD basic
  * algorithms
- * 
- * @author Dmitriy
- * 
  */
 public class SSVDPrototype {
 
-
-  private Omega omega;
-  private int kp; // k+p
+  private final Omega omega;
+  private final int kp; // k+p
   private GivensThinSolver qSolver;
-  private double[] yRow;
+  private final double[] yRow;
   private int cnt;
   private int blckCnt;
-  private int r;
-  private List<UpperTriangular> rBlocks = new ArrayList<UpperTriangular>();
-  private List<double[][]> qtBlocks = new ArrayList<double[][]>();
-  private List<double[]> yLookahead;
+  private final int r;
+  private final List<UpperTriangular> rBlocks = new ArrayList<UpperTriangular>();
+  private final List<double[][]> qtBlocks = new ArrayList<double[][]>();
+  private final List<double[]> yLookahead;
 
   public SSVDPrototype(long seed, int kp, int r) {
-    super();
     this.kp = kp;
     omega = new Omega(seed, kp / 2, kp - (kp / 2));
     yRow = new double[kp];
@@ -62,7 +58,7 @@ public class SSVDPrototype {
     yLookahead = new ArrayList<double[]>(kp);
   }
 
-  void firstPass(int aRowId, Vector aRow) throws IOException {
+  void firstPass(Vector aRow) {
 
     omega.computeYRow(aRow, yRow);
 
@@ -72,8 +68,9 @@ public class SSVDPrototype {
 
     while (yLookahead.size() > kp) {
 
-      if (qSolver == null)
+      if (qSolver == null) {
         qSolver = new GivensThinSolver(r, kp);
+      }
 
       qSolver.appendRow(yLookahead.remove(0));
       if (qSolver.isFull()) {
@@ -90,14 +87,16 @@ public class SSVDPrototype {
 
   void finishFirstPass() {
 
-    if (qSolver == null && yLookahead.size() == 0)
+    if (qSolver == null && yLookahead.isEmpty()) {
       return;
-    if (qSolver == null)
+    }
+    if (qSolver == null) {
       qSolver = new GivensThinSolver(yLookahead.size(), kp);
+    }
     // grow q solver up if necessary
 
     qSolver.adjust(qSolver.getCnt() + yLookahead.size());
-    while (yLookahead.size() > 0) {
+    while (!yLookahead.isEmpty()) {
 
       qSolver.appendRow(yLookahead.remove(0));
       if (qSolver.isFull()) {
@@ -111,14 +110,16 @@ public class SSVDPrototype {
     }
 
     // simulate reducers -- produce qHats
-    for (int i = 0; i < rBlocks.size(); i++)
-      qtBlocks.set(i, GivensThinSolver.computeQtHat(qtBlocks.get(i), i,
-          new DeepCopyIterator(rBlocks.listIterator())));
+    for (int i = 0; i < rBlocks.size(); i++) {
+      qtBlocks.set(i,
+                   GivensThinSolver.computeQtHat(qtBlocks.get(i), i,
+                                                 new CopyConstructorIterator<UpperTriangular>(rBlocks.listIterator())));
+    }
     cnt = 0;
     blckCnt = 0;
   }
 
-  void secondPass(int aRowId, Vector aRow, PartialRowEmitter btEmitter)
+  void secondPass(Vector aRow, PartialRowEmitter btEmitter)
     throws IOException {
     int n = aRow.size();
     double[][] qtHat = qtBlocks.get(blckCnt);
@@ -127,8 +128,9 @@ public class SSVDPrototype {
     int qRowBlckIndex = r - cnt - 1; // <-- reverse order since we fed A in
                                        // reverse
     double[] qRow = new double[kp];
-    for (int i = 0; i < kp; i++)
+    for (int i = 0; i < kp; i++) {
       qRow[i] = qtHat[i][qRowBlckIndex];
+    }
     Vector qRowV = new DenseVector(qRow, true);
 
     if (++cnt == r) {
@@ -136,37 +138,14 @@ public class SSVDPrototype {
       cnt = 0;
     }
 
-    for (int i = 0; i < n; i++)
+    for (int i = 0; i < n; i++) {
       btEmitter.emitRow(i, qRowV.times(aRow.getQuick(i)));
-
-  }
-
-  private static class DeepCopyIterator implements Iterator<UpperTriangular> {
-
-    private Iterator<UpperTriangular> delegate;
-
-    public DeepCopyIterator(Iterator<UpperTriangular> del) {
-      super();
-      delegate = del;
-    }
-
-    public boolean hasNext() {
-      return delegate.hasNext();
-    }
-
-    public UpperTriangular next() {
-
-      return new UpperTriangular(delegate.next());
-    }
-
-    public void remove() {
-      delegate.remove();
     }
 
   }
 
-  public static void testThinQr(int dims, int kp, final long rndSeed)
-    throws Exception {
+
+  public static void testThinQr(int dims, int kp) throws IOException {
 
     DenseMatrix mx = new DenseMatrix(dims << 2, dims);
     // mx.assign(new UnaryFunction() {
@@ -179,10 +158,12 @@ public class SSVDPrototype {
     // }
     // });
 
-    Random rnd = new Random();
-    for (int i = 0; i < mx.rowSize(); i++)
-      for (int j = 0; j < mx.columnSize(); j++)
+    Random rnd = RandomUtils.getRandom();
+    for (int i = 0; i < mx.rowSize(); i++) {
+      for (int j = 0; j < mx.columnSize(); j++) {
         mx.set(i, j, rnd.nextDouble() * 1000);
+      }
+    }
 
     mx.setQuick(0, 0, 1);
     mx.setQuick(0, 1, 2);
@@ -197,34 +178,38 @@ public class SSVDPrototype {
     SingularValueDecomposition svd2 = new SingularValueDecomposition(mx);
     double[] svaluesControl = svd2.getSingularValues();
 
-    for (int i = 0; i < kp; i++)
+    for (int i = 0; i < kp; i++) {
       System.out.printf("%.3e ", svaluesControl[i]);
+    }
     System.out.println();
 
     int m = mx.rowSize(); /* ,n=mx.columnSize(); */
 
-    long seed = new Random().nextLong();
+    long seed = RandomUtils.getRandom().nextLong();
 
     final Map<Integer, Vector> btRows = new HashMap<Integer, Vector>();
 
     PartialRowEmitter btEmitter = new PartialRowEmitter() {
       @Override
-      public void emitRow(int rowNum, Vector row) throws IOException {
+      public void emitRow(int rowNum, Vector row) {
         Vector btRow = btRows.get(rowNum);
-        if (btRow != null)
+        if (btRow != null) {
           row.addTo(btRow);
+        }
         btRows.put(rowNum, btRow == null ? new DenseVector(row) : btRow);
       }
     };
 
     SSVDPrototype mapperSimulation = new SSVDPrototype(seed, kp, 3000);
-    for (int i = 0; i < m; i++)
-      mapperSimulation.firstPass(i, mx.getRow(i));
+    for (int i = 0; i < m; i++) {
+      mapperSimulation.firstPass(mx.getRow(i));
+    }
 
     mapperSimulation.finishFirstPass();
 
-    for (int i = 0; i < m; i++)
-      mapperSimulation.secondPass(i, mx.getRow(i), btEmitter);
+    for (int i = 0; i < m; i++) {
+      mapperSimulation.secondPass(mx.getRow(i), btEmitter);
+    }
 
     // LocalSSVDTest.assertOrthonormality(mapperSimulation.m_qt.transpose(),
     // false,1e-10);
@@ -234,10 +219,11 @@ public class SSVDPrototype {
     PartialRowEmitter bbtEmitter = new PartialRowEmitter() {
 
       @Override
-      public void emitRow(int rowNum, Vector row) throws IOException {
+      public void emitRow(int rowNum, Vector row) {
         Vector bbtRow = bbt.get(rowNum);
-        if (bbtRow != null)
+        if (bbtRow != null) {
           row.addTo(bbtRow);
+        }
         bbt.put(rowNum, bbtRow == null ? new DenseVector(row) : bbtRow);
       }
     };
@@ -245,32 +231,35 @@ public class SSVDPrototype {
     for (Map.Entry<Integer, Vector> btRowEntry : btRows.entrySet()) {
       Vector btRow = btRowEntry.getValue();
       assert btRow.size() == kp;
-      for (int i = 0; i < kp; i++)
+      for (int i = 0; i < kp; i++) {
         bbtEmitter.emitRow(i, btRow.times(btRow.getQuick(i)));
+      }
     }
 
     double[][] bbtValues = new double[kp][];
     for (int i = 0; i < kp; i++) {
       bbtValues[i] = new double[kp];
       Vector bbtRow = bbt.get(i);
-      for (int j = 0; j < kp; j++)
+      for (int j = 0; j < kp; j++) {
         bbtValues[i][j] = bbtRow.getQuick(j);
+      }
     }
 
     EigenSolverWrapper eigenWrapper = new EigenSolverWrapper(bbtValues);
     double[] eigenva2 = eigenWrapper.getEigenValues();
     double[] svalues = new double[kp];
-    for (int i = 0; i < kp; i++)
+    for (int i = 0; i < kp; i++) {
       svalues[i] = Math.sqrt(eigenva2[i]); // sqrt?
+    }
 
-    for (int i = 0; i < kp; i++)
+    for (int i = 0; i < kp; i++) {
       System.out.printf("%.3e ", svalues[i]);
+    }
     System.out.println();
 
   }
 
-  public static void testBlockQrWithSSVD(int dims, int kp, int r,
-      final long rndSeed) throws Exception {
+  public static void testBlockQrWithSSVD(int dims, int kp, int r, long rndSeed) throws IOException {
 
     DenseMatrix mx = new DenseMatrix(dims << 2, dims);
     // mx.assign(new UnaryFunction() {
@@ -283,10 +272,12 @@ public class SSVDPrototype {
     // }
     // });
 
-    Random rnd = new Random();
-    for (int i = 0; i < mx.rowSize(); i++)
-      for (int j = 0; j < mx.columnSize(); j++)
+    Random rnd = RandomUtils.getRandom();
+    for (int i = 0; i < mx.rowSize(); i++) {
+      for (int j = 0; j < mx.columnSize(); j++) {
         mx.set(i, j, (rnd.nextDouble() - 0.5) * 1000);
+      }
+    }
     mx.setQuick(0, 0, 1);
     mx.setQuick(0, 1, 2);
     mx.setQuick(0, 2, 3);
@@ -300,8 +291,9 @@ public class SSVDPrototype {
     SingularValueDecomposition svd2 = new SingularValueDecomposition(mx);
     double[] svaluesControl = svd2.getSingularValues();
 
-    for (int i = 0; i < kp; i++)
+    for (int i = 0; i < kp; i++) {
       System.out.printf("%e ", svaluesControl[i]);
+    }
     System.out.println();
 
     int m = mx.rowSize(); /* ,n=mx.columnSize(); */
@@ -310,22 +302,25 @@ public class SSVDPrototype {
 
     PartialRowEmitter btEmitter = new PartialRowEmitter() {
       @Override
-      public void emitRow(int rowNum, Vector row) throws IOException {
+      public void emitRow(int rowNum, Vector row) {
         Vector btRow = btRows.get(rowNum);
-        if (btRow != null)
+        if (btRow != null) {
           row.addTo(btRow);
+        }
         btRows.put(rowNum, btRow == null ? new DenseVector(row) : btRow);
       }
     };
 
     SSVDPrototype mapperSimulation = new SSVDPrototype(rndSeed, kp, r);
-    for (int i = 0; i < m; i++)
-      mapperSimulation.firstPass(i, mx.getRow(i));
+    for (int i = 0; i < m; i++) {
+      mapperSimulation.firstPass(mx.getRow(i));
+    }
 
     mapperSimulation.finishFirstPass();
 
-    for (int i = 0; i < m; i++)
-      mapperSimulation.secondPass(i, mx.getRow(i), btEmitter);
+    for (int i = 0; i < m; i++) {
+      mapperSimulation.secondPass(mx.getRow(i), btEmitter);
+    }
 
     // LocalSSVDTest.assertOrthonormality(mapperSimulation.m_qt.transpose(),
     // false,1e-10);
@@ -335,10 +330,11 @@ public class SSVDPrototype {
     PartialRowEmitter bbtEmitter = new PartialRowEmitter() {
 
       @Override
-      public void emitRow(int rowNum, Vector row) throws IOException {
+      public void emitRow(int rowNum, Vector row) {
         Vector bbtRow = bbt.get(rowNum);
-        if (bbtRow != null)
+        if (bbtRow != null) {
           row.addTo(bbtRow);
+        }
         bbt.put(rowNum, bbtRow == null ? new DenseVector(row) : bbtRow);
       }
     };
@@ -346,33 +342,37 @@ public class SSVDPrototype {
     for (Map.Entry<Integer, Vector> btRowEntry : btRows.entrySet()) {
       Vector btRow = btRowEntry.getValue();
       assert btRow.size() == kp;
-      for (int i = 0; i < kp; i++)
+      for (int i = 0; i < kp; i++) {
         bbtEmitter.emitRow(i, btRow.times(btRow.getQuick(i)));
+      }
     }
 
     double[][] bbtValues = new double[kp][];
     for (int i = 0; i < kp; i++) {
       bbtValues[i] = new double[kp];
       Vector bbtRow = bbt.get(i);
-      for (int j = 0; j < kp; j++)
+      for (int j = 0; j < kp; j++) {
         bbtValues[i][j] = bbtRow.getQuick(j);
+      }
     }
 
     EigenSolverWrapper eigenWrapper = new EigenSolverWrapper(bbtValues);
 
     double[] eigenva2 = eigenWrapper.getEigenValues();
     double[] svalues = new double[kp];
-    for (int i = 0; i < kp; i++)
+    for (int i = 0; i < kp; i++) {
       svalues[i] = Math.sqrt(eigenva2[i]); // sqrt?
+    }
 
-    for (int i = 0; i < kp; i++)
+    for (int i = 0; i < kp; i++) {
       System.out.printf("%e ", svalues[i]);
+    }
     System.out.println();
   }
 
   public static void main(String[] args) throws Exception {
     // testThinQr();
-    long seed = new Random().nextLong();
+    long seed = RandomUtils.getRandom().nextLong();
     testBlockQrWithSSVD(200, 200, 800, seed);
     testBlockQrWithSSVD(200, 20, 800, seed);
     testBlockQrWithSSVD(200, 20, 850, seed); // test trimming

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java Thu Mar 31 09:25:25 2011
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.Deque;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Random;
@@ -37,6 +38,8 @@ import org.apache.hadoop.io.SequenceFile
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.io.Writable;
 import org.apache.mahout.common.IOUtils;
+import org.apache.mahout.common.RandomUtils;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileValueIterable;
 import org.apache.mahout.math.DenseVector;
 import org.apache.mahout.math.Vector;
 import org.apache.mahout.math.VectorWritable;
@@ -86,9 +89,6 @@ import org.apache.mahout.math.ssvd.Eigen
  * containing n x k V matrix file(s).
  * 
  * </UL>
- * 
- * 
- * 
  */
 public class SSVDSolver {
 
@@ -99,13 +99,13 @@ public class SSVDSolver {
   private String vPath;
 
   // configured stuff
-  private Configuration conf;
-  private Path[] inputPath;
-  private Path outputPath;
-  private int ablockRows;
-  private int k;
-  private int p;
-  private int reduceTasks;
+  private final Configuration conf;
+  private final Path[] inputPath;
+  private final Path outputPath;
+  private final int ablockRows;
+  private final int k;
+  private final int p;
+  private final int reduceTasks;
   private int minSplitSize = -1;
   private boolean cUHalfSigma;
   private boolean cVHalfSigma;
@@ -135,9 +135,13 @@ public class SSVDSolver {
    * @throws IOException
    *           when IO condition occurs.
    */
-
-  public SSVDSolver(Configuration conf, Path[] inputPath, Path outputPath,
-      int ablockRows, int k, int p, int reduceTasks) throws IOException {
+  public SSVDSolver(Configuration conf,
+                    Path[] inputPath,
+                    Path outputPath,
+                    int ablockRows,
+                    int k,
+                    int p,
+                    int reduceTasks) {
     this.conf = conf;
     this.inputPath = inputPath;
     this.outputPath = outputPath;
@@ -235,10 +239,9 @@ public class SSVDSolver {
    */
   public void run() throws IOException {
 
-    LinkedList<Closeable> closeables = new LinkedList<Closeable>();
+    Deque<Closeable> closeables = new LinkedList<Closeable>();
     try {
-      Class<? extends Writable> labelType = sniffInputLabelType(inputPath,
-          conf, closeables);
+      Class<? extends Writable> labelType = sniffInputLabelType(inputPath, conf);
       FileSystem fs = FileSystem.get(conf);
 
       Path qPath = new Path(outputPath, "Q-job");
@@ -253,30 +256,30 @@ public class SSVDSolver {
         fs.delete(outputPath, true);
       }
 
-      Random rnd = new Random();
+      Random rnd = RandomUtils.getRandom();
       long seed = rnd.nextLong();
 
-      QJob.run(conf, inputPath, qPath, ablockRows, minSplitSize, k, p, seed,
-          reduceTasks);
+      QJob.run(conf, inputPath, qPath, ablockRows, minSplitSize, k, p, seed, reduceTasks);
 
-      BtJob.run(conf, inputPath, qPath, btPath, minSplitSize, k, p,
-          reduceTasks, labelType);
+      BtJob.run(conf, inputPath, qPath, btPath, minSplitSize, k, p, reduceTasks, labelType);
 
       BBtJob.run(conf, new Path(btPath, BtJob.OUTPUT_BT + "-*"), bbtPath, 1);
 
-      UpperTriangular bbt = loadUpperTriangularMatrix(fs, new Path(bbtPath,
-          BBtJob.OUTPUT_BBT + "-*"), conf);
+      UpperTriangular bbt = loadUpperTriangularMatrix(fs, new Path(bbtPath, BBtJob.OUTPUT_BBT + "-*"), conf);
 
       // convert bbt to something our eigensolver could understand
       assert bbt.columnSize() == k + p;
 
       double[][] bbtSquare = new double[k + p][];
-      for (int i = 0; i < k + p; i++)
+      for (int i = 0; i < k + p; i++) {
         bbtSquare[i] = new double[k + p];
+      }
 
-      for (int i = 0; i < k + p; i++)
-        for (int j = i; j < k + p; j++)
+      for (int i = 0; i < k + p; i++) {
+        for (int j = i; j < k + p; j++) {
           bbtSquare[i][j] = bbtSquare[j][i] = bbt.getQuick(i, j);
+        }
+      }
 
       svalues = new double[k + p];
 
@@ -284,8 +287,9 @@ public class SSVDSolver {
       EigenSolverWrapper eigenWrapper = new EigenSolverWrapper(bbtSquare);
 
       double[] eigenva2 = eigenWrapper.getEigenValues();
-      for (int i = 0; i < k + p; i++)
+      for (int i = 0; i < k + p; i++) {
         svalues[i] = Math.sqrt(eigenva2[i]); // sqrt?
+      }
 
       // save/redistribute UHat
       //
@@ -322,17 +326,21 @@ public class SSVDSolver {
       svWriter.close();
 
       UJob ujob = null;
+      if (computeU) {
+        ujob = new UJob();
+        ujob.start(conf,
+                   new Path(btPath, BtJob.OUTPUT_Q + "-*"),
+                   uHatPath, svPath, uPath, k, reduceTasks, labelType, cUHalfSigma);
+                                  // actually this is  map-only job anyway
+      }
+
       VJob vjob = null;
-      if (computeU)
-        (ujob = new UJob()).start(conf,
-            new Path(btPath, BtJob.OUTPUT_Q + "-*"), uHatPath, svPath, uPath,
-            k, reduceTasks, labelType, cUHalfSigma); // actually this is
-                                                     // map-only job anyway
-
-      if (computeV)
-        (vjob = new VJob()).start(conf,
-            new Path(btPath, BtJob.OUTPUT_BT + "-*"), uHatPath, svPath, vPath,
-            k, reduceTasks, cVHalfSigma);
+      if (computeV) {
+        vjob = new VJob();
+        vjob.start(conf,
+                   new Path(btPath, BtJob.OUTPUT_BT + "-*"),
+                   uHatPath, svPath, vPath, k, reduceTasks, cVHalfSigma);
+      }
 
       if (ujob != null) {
         ujob.waitForCompletion();
@@ -354,49 +362,40 @@ public class SSVDSolver {
 
   }
 
-  private static Class<? extends Writable> sniffInputLabelType(
-      Path[] inputPath, Configuration conf, LinkedList<Closeable> closeables)
+  private static Class<? extends Writable> sniffInputLabelType(Path[] inputPath, Configuration conf)
     throws IOException {
     FileSystem fs = FileSystem.get(conf);
     for (Path p : inputPath) {
       FileStatus[] fstats = fs.globStatus(p);
-      if (fstats == null || fstats.length == 0)
+      if (fstats == null || fstats.length == 0) {
         continue;
-      SequenceFile.Reader r = new SequenceFile.Reader(fs, fstats[0].getPath(),
-          conf);
-      closeables.addFirst(r);
-
+      }
+      SequenceFile.Reader r = new SequenceFile.Reader(fs, fstats[0].getPath(), conf);
       try {
         return r.getKeyClass().asSubclass(Writable.class);
       } finally {
-        closeables.remove(r);
         r.close();
       }
     }
-
-    throw new IOException(
-        "Unable to open input files to determine input label type.");
+    throw new IOException("Unable to open input files to determine input label type.");
   }
 
-  private static final Pattern OUTPUT_FILE_PATTERN = Pattern
-      .compile("(\\w+)-(m|r)-(\\d+)(\\.\\w+)?");
+  private static final Pattern OUTPUT_FILE_PATTERN = Pattern.compile("(\\w+)-(m|r)-(\\d+)(\\.\\w+)?");
 
-  static Comparator<FileStatus> partitionComparator = new Comparator<FileStatus>() {
-    private Matcher matcher = OUTPUT_FILE_PATTERN.matcher("");
+  static final Comparator<FileStatus> partitionComparator = new Comparator<FileStatus>() {
+    private final Matcher matcher = OUTPUT_FILE_PATTERN.matcher("");
 
     @Override
     public int compare(FileStatus o1, FileStatus o2) {
       matcher.reset(o1.getPath().getName());
-      if (!matcher.matches())
-        throw new RuntimeException(String.format(
-            "Unexpected file name, unable to deduce partition #:%s", o1
-                .getPath().toString()));
+      if (!matcher.matches()) {
+        throw new IllegalArgumentException("Unexpected file name, unable to deduce partition #:" + o1.getPath());
+      }
       int p1 = Integer.parseInt(matcher.group(3));
       matcher.reset(o2.getPath().getName());
-      if (!matcher.matches())
-        throw new RuntimeException(String.format(
-            "Unexpected file name, unable to deduce partition #:%s", o2
-                .getPath().toString()));
+      if (!matcher.matches()) {
+        throw new IllegalArgumentException("Unexpected file name, unable to deduce partition #:" + o2.getPath());
+      }
 
       int p2 = Integer.parseInt(matcher.group(3));
       return p1 - p2;
@@ -418,16 +417,14 @@ public class SSVDSolver {
    * @throws IOException
    *           when I/O occurs.
    */
-  public static double[][] loadDistributedRowMatrix(FileSystem fs, Path glob,
-      Configuration conf) throws IOException {
+  public static double[][] loadDistributedRowMatrix(FileSystem fs, Path glob, Configuration conf) throws IOException {
 
     FileStatus[] files = fs.globStatus(glob);
-    if (files == null)
+    if (files == null) {
       return null;
+    }
 
     List<double[]> denseData = new ArrayList<double[]>();
-    IntWritable iw = new IntWritable();
-    VectorWritable vw = new VectorWritable();
 
     // int m=0;
 
@@ -436,22 +433,15 @@ public class SSVDSolver {
     Arrays.sort(files, partitionComparator);
 
     for (FileStatus fstat : files) {
-      Path file = fstat.getPath();
-      SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
-      try {
-        while (reader.next(iw, vw)) {
-          Vector v = vw.get();
-          int size;
-          double[] row = new double[size = v.size()];
-          for (int i = 0; i < size; i++)
-            row[i] = v.get(i);
-          // ignore row label.
-          // int rowIndex=iw.get();
-          denseData.add(row);
-
+      for (VectorWritable value : new SequenceFileValueIterable<VectorWritable>(fstat.getPath(), true, conf)) {
+        Vector v = value.get();
+        int size = v.size();
+        double[] row = new double[size];
+        for (int i = 0; i < size; i++) {
+          row[i] = v.get(i);
         }
-      } finally {
-        reader.close();
+        // ignore row label.
+        denseData.add(row);
       }
     }
 
@@ -462,37 +452,29 @@ public class SSVDSolver {
       Path glob, Configuration conf) throws IOException {
 
     FileStatus[] files = fs.globStatus(glob);
-    if (files == null)
+    if (files == null) {
       return null;
-
-    IntWritable iw = new IntWritable();
-    VectorWritable vw = new VectorWritable();
-    UpperTriangular result = null;
+    }
 
     // assume it is partitioned output, so we need to read them up
     // in order of partitions.
     Arrays.sort(files, partitionComparator);
 
+    UpperTriangular result = null;
     for (FileStatus fstat : files) {
-      Path file = fstat.getPath();
-      SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
-      try {
-        while (reader.next(iw, vw)) {
-          Vector v = vw.get();
-          if (result == null)
-            result = new UpperTriangular(v);
-          else
-            throw new IOException(
-                "Unexpected overrun in upper triangular matrix files");
+      for (VectorWritable value : new SequenceFileValueIterable<VectorWritable>(fstat.getPath(), true, conf)) {
+        Vector v = value.get();
+        if (result == null) {
+          result = new UpperTriangular(v);
+        } else {
+          throw new IOException("Unexpected overrun in upper triangular matrix files");
         }
-      } finally {
-        reader.close();
       }
     }
 
-    if (result == null)
-      throw new IOException(
-          "Unexpected underrun in upper triangular matrix files");
+    if (result == null) {
+      throw new IOException("Unexpected underrun in upper triangular matrix files");
+    }
     return result;
   }
 

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java Thu Mar 31 09:25:25 2011
@@ -20,15 +20,14 @@ package org.apache.mahout.vectorizer;
 import java.io.IOException;
 import java.net.URI;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
@@ -41,7 +40,10 @@ import org.apache.hadoop.mapreduce.lib.i
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
 import org.apache.mahout.common.HadoopUtil;
+import org.apache.mahout.common.Pair;
 import org.apache.mahout.common.StringTuple;
+import org.apache.mahout.common.iterator.sequencefile.PathType;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileDirIterable;
 import org.apache.mahout.math.VectorWritable;
 import org.apache.mahout.vectorizer.collocations.llr.CollocDriver;
 import org.apache.mahout.vectorizer.common.PartialVectorMerger;
@@ -152,19 +154,22 @@ public final class DictionaryVectorizer 
     List<Path> dictionaryChunks;
     if (maxNGramSize == 1) {
       startWordCounting(input, dictionaryJobPath, baseConf, minSupport);
-      dictionaryChunks = createDictionaryChunks(dictionaryJobPath, output,
-        baseConf, chunkSizeInMegabytes, new LongWritable(), maxTermDimension);
+      dictionaryChunks =
+          createDictionaryChunks(dictionaryJobPath, output, baseConf, chunkSizeInMegabytes, maxTermDimension);
     } else {
       CollocDriver.generateAllGrams(input, dictionaryJobPath, baseConf, maxNGramSize,
         minSupport, minLLRValue, numReducers);
-      dictionaryChunks = createDictionaryChunks(new Path(
-          new Path(output, DICTIONARY_JOB_FOLDER), CollocDriver.NGRAM_OUTPUT_DIRECTORY), output,
-          baseConf,
-        chunkSizeInMegabytes, new DoubleWritable(), maxTermDimension);
+      dictionaryChunks =
+          createDictionaryChunks(new Path(new Path(output, DICTIONARY_JOB_FOLDER),
+                                          CollocDriver.NGRAM_OUTPUT_DIRECTORY),
+                                 output,
+                                 baseConf,
+                                 chunkSizeInMegabytes,
+                                 maxTermDimension);
     }
     
     int partialVectorIndex = 0;
-    List<Path> partialVectorPaths = new ArrayList<Path>();
+    Collection<Path> partialVectorPaths = new ArrayList<Path>();
     for (Path dictionaryChunk : dictionaryChunks) {
       Path partialVectorOutputPath = new Path(output, VECTOR_OUTPUT_FOLDER + partialVectorIndex++);
       partialVectorPaths.add(partialVectorOutputPath);
@@ -173,12 +178,11 @@ public final class DictionaryVectorizer 
     }
     
     Configuration conf = new Configuration(baseConf);
-    FileSystem fs = FileSystem.get(partialVectorPaths.get(0).toUri(), conf);
-    
+
     Path outputDir = new Path(output, DOCUMENT_VECTOR_OUTPUT_FOLDER);
     PartialVectorMerger.mergePartialVectors(partialVectorPaths, outputDir, conf, normPower, logNormalize,
       maxTermDimension[0], sequentialAccess, namedVectors, numReducers);
-    HadoopUtil.deletePaths(partialVectorPaths, fs);  
+    HadoopUtil.delete(conf, partialVectorPaths);
   }
   
   /**
@@ -189,48 +193,40 @@ public final class DictionaryVectorizer 
                                                    Path dictionaryPathBase,
                                                    Configuration baseConf,
                                                    int chunkSizeInMegabytes,
-                                                   Writable value,
                                                    int[] maxTermDimension) throws IOException {
     List<Path> chunkPaths = new ArrayList<Path>();
     
-    Writable key = new Text();
     Configuration conf = new Configuration(baseConf);
     
     FileSystem fs = FileSystem.get(wordCountPath.toUri(), conf);
-    FileStatus[] outputFiles = fs.globStatus(new Path(wordCountPath, OUTPUT_FILES_PATTERN));
-    
+
     long chunkSizeLimit = chunkSizeInMegabytes * 1024L * 1024L;
     int chunkIndex = 0;
     Path chunkPath = new Path(dictionaryPathBase, DICTIONARY_FILE + chunkIndex);
     chunkPaths.add(chunkPath);
     
-    SequenceFile.Writer dictWriter = new SequenceFile.Writer(fs, conf, chunkPath, Text.class,
-        IntWritable.class);
+    SequenceFile.Writer dictWriter = new SequenceFile.Writer(fs, conf, chunkPath, Text.class, IntWritable.class);
     
     long currentChunkSize = 0;
-    
+    Path filesPattern = new Path(wordCountPath, OUTPUT_FILES_PATTERN);
     int i = 0;
-    for (FileStatus fileStatus : outputFiles) {
-      Path path = fileStatus.getPath();
-      SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
-      // key is feature value is count
-      while (reader.next(key, value)) {
-        if (currentChunkSize > chunkSizeLimit) {
-          dictWriter.close();
-          chunkIndex++;
-          
-          chunkPath = new Path(dictionaryPathBase, DICTIONARY_FILE + chunkIndex);
-          chunkPaths.add(chunkPath);
-          
-          dictWriter = new SequenceFile.Writer(fs, conf, chunkPath, Text.class, IntWritable.class);
-          currentChunkSize = 0;
-        }
-        
-        int fieldSize = DICTIONARY_BYTE_OVERHEAD + key.toString().length() * 2
-                        + Integer.SIZE / 8;
-        currentChunkSize += fieldSize;
-        dictWriter.append(key, new IntWritable(i++));
+    for (Pair<Writable,Writable> record :
+         new SequenceFileDirIterable<Writable,Writable>(filesPattern, PathType.GLOB, null, null, true, conf)) {
+      if (currentChunkSize > chunkSizeLimit) {
+        dictWriter.close();
+        chunkIndex++;
+
+        chunkPath = new Path(dictionaryPathBase, DICTIONARY_FILE + chunkIndex);
+        chunkPaths.add(chunkPath);
+
+        dictWriter = new SequenceFile.Writer(fs, conf, chunkPath, Text.class, IntWritable.class);
+        currentChunkSize = 0;
       }
+
+      Writable key = record.getFirst();
+      int fieldSize = DICTIONARY_BYTE_OVERHEAD + key.toString().length() * 2 + Integer.SIZE / 8;
+      currentChunkSize += fieldSize;
+      dictWriter.append(key, new IntWritable(i++));
     }
     maxTermDimension[0] = i;
     dictWriter.close();
@@ -300,7 +296,7 @@ public final class DictionaryVectorizer 
     job.setOutputFormatClass(SequenceFileOutputFormat.class);
     job.setNumReduceTasks(numReducers);
 
-    HadoopUtil.overwriteOutput(output);
+    HadoopUtil.delete(conf, output);
     
     job.waitForCompletion(true);
   }
@@ -336,7 +332,7 @@ public final class DictionaryVectorizer 
     job.setReducerClass(TermCountReducer.class);
     job.setOutputFormatClass(SequenceFileOutputFormat.class);
     
-    HadoopUtil.overwriteOutput(output);
+    HadoopUtil.delete(conf, output);
     
     job.waitForCompletion(true);
   }

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DocumentProcessor.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DocumentProcessor.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DocumentProcessor.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DocumentProcessor.java Thu Mar 31 09:25:25 2011
@@ -89,7 +89,7 @@ public final class DocumentProcessor {
     job.setInputFormatClass(SequenceFileInputFormat.class);
     job.setNumReduceTasks(0);
     job.setOutputFormatClass(SequenceFileOutputFormat.class);
-    HadoopUtil.overwriteOutput(output);
+    HadoopUtil.delete(conf, output);
 
     job.waitForCompletion(true);
   }

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java Thu Mar 31 09:25:25 2011
@@ -52,6 +52,7 @@ public final class SparseVectorsFromSequ
     ToolRunner.run(new SparseVectorsFromSequenceFiles(), args);
   }
   
+  @Override
   public int run(String[] args) throws Exception {
     DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
     ArgumentBuilder abuilder = new ArgumentBuilder();
@@ -172,7 +173,7 @@ public final class SparseVectorsFromSequ
       log.info("Maximum n-gram size is: {}", maxNGramSize);
       
       if (cmdLine.hasOption(overwriteOutput)) {
-        HadoopUtil.overwriteOutput(outputDir);
+        HadoopUtil.delete(getConf(), outputDir);
       }
       
       float minLLRValue = LLRReducer.DEFAULT_MIN_LLR;
@@ -234,9 +235,9 @@ public final class SparseVectorsFromSequ
       if (cmdLine.hasOption(logNormalizeOpt)) {
         logNormalize = true;
       }
-      
-      HadoopUtil.overwriteOutput(outputDir);
+
       Configuration conf = getConf();
+      HadoopUtil.delete(conf, outputDir);
       Path tokenizedPath = new Path(outputDir, DocumentProcessor.TOKENIZED_DOCUMENT_OUTPUT_FOLDER);
       DocumentProcessor.tokenizeDocuments(inputDir, analyzerClass, tokenizedPath, conf);
       

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java Thu Mar 31 09:25:25 2011
@@ -106,7 +106,7 @@ public final class CollocDriver extends 
     log.info("Maximum n-gram size is: {}", maxNGramSize);
 
     if (argMap.containsKey("--overwrite")) {
-      HadoopUtil.overwriteOutput(output);
+      HadoopUtil.delete(getConf(), output);
     }
 
     int minSupport = CollocReducer.DEFAULT_MIN_SUPPORT;

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/common/PartialVectorMerger.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/common/PartialVectorMerger.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/common/PartialVectorMerger.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/common/PartialVectorMerger.java Thu Mar 31 09:25:25 2011
@@ -124,7 +124,7 @@ public final class PartialVectorMerger {
     job.setOutputFormatClass(SequenceFileOutputFormat.class);
     job.setNumReduceTasks(numReducers);
 
-    HadoopUtil.overwriteOutput(output);
+    HadoopUtil.delete(conf, output);
 
     job.waitForCompletion(true);
   }

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java Thu Mar 31 09:25:25 2011
@@ -24,16 +24,16 @@ import java.util.Iterator;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.lucene.analysis.shingle.ShingleFilter;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.mahout.common.Pair;
 import org.apache.mahout.common.StringTuple;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileIterable;
 import org.apache.mahout.math.NamedVector;
 import org.apache.mahout.math.RandomAccessSparseVector;
 import org.apache.mahout.math.SequentialAccessSparseVector;
@@ -122,14 +122,10 @@ public class TFPartialVectorReducer exte
     maxNGramSize = conf.getInt(DictionaryVectorizer.MAX_NGRAMS, maxNGramSize);
 
     Path dictionaryFile = new Path(localFiles[0].getPath());
-    FileSystem fs = dictionaryFile.getFileSystem(conf);
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, dictionaryFile, conf);
-    Writable key = new Text();
-    IntWritable value = new IntWritable();
-
     // key is word value is id
-    while (reader.next(key, value)) {
-      dictionary.put(key.toString(), value.get());
+    for (Pair<Writable,IntWritable> record :
+         new SequenceFileIterable<Writable,IntWritable>(dictionaryFile, true, conf)) {
+      dictionary.put(record.getFirst().toString(), record.getSecond().get());
     }
   }
 

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFConverter.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFConverter.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFConverter.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFConverter.java Thu Mar 31 09:25:25 2011
@@ -25,7 +25,6 @@ import java.util.List;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
@@ -40,6 +39,8 @@ import org.apache.hadoop.mapreduce.lib.o
 import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
 import org.apache.mahout.common.HadoopUtil;
 import org.apache.mahout.common.Pair;
+import org.apache.mahout.common.iterator.sequencefile.PathType;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileDirIterable;
 import org.apache.mahout.math.VectorWritable;
 import org.apache.mahout.vectorizer.common.PartialVectorMerger;
 import org.apache.mahout.vectorizer.term.TermDocumentCountMapper;
@@ -169,7 +170,6 @@ public final class TFIDFConverter {
     }
 
     Configuration conf = new Configuration(baseConf);
-    FileSystem fs = FileSystem.get(partialVectorPaths.get(0).toUri(), conf);
 
     Path outputDir = new Path(output, DOCUMENT_VECTOR_OUTPUT_FOLDER);
     
@@ -182,7 +182,7 @@ public final class TFIDFConverter {
                                             sequentialAccessOutput,
                                             namedVector,
                                             numReducers);
-    HadoopUtil.deletePaths(partialVectorPaths, fs);
+    HadoopUtil.delete(conf, partialVectorPaths);
 
   }
 
@@ -195,13 +195,9 @@ public final class TFIDFConverter {
                                                                  Configuration baseConf,
                                                                  int chunkSizeInMegabytes) throws IOException {
     List<Path> chunkPaths = new ArrayList<Path>();
-
-    IntWritable key = new IntWritable();
-    LongWritable value = new LongWritable();
     Configuration conf = new Configuration(baseConf);
 
     FileSystem fs = FileSystem.get(featureCountPath.toUri(), conf);
-    FileStatus[] outputFiles = fs.globStatus(new Path(featureCountPath, OUTPUT_FILES_PATTERN));
 
     long chunkSizeLimit = chunkSizeInMegabytes * 1024L * 1024L;
     int chunkIndex = 0;
@@ -213,34 +209,34 @@ public final class TFIDFConverter {
     long currentChunkSize = 0;
     long featureCount = 0;
     long vectorCount = Long.MAX_VALUE;
-    for (FileStatus fileStatus : outputFiles) {
-      Path path = fileStatus.getPath();
-      SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
-      // key is feature value is count
-      while (reader.next(key, value)) {
-        if (currentChunkSize > chunkSizeLimit) {
-          freqWriter.close();
-          chunkIndex++;
-
-          chunkPath = new Path(dictionaryPathBase, FREQUENCY_FILE + chunkIndex);
-          chunkPaths.add(chunkPath);
-
-          freqWriter = new SequenceFile.Writer(fs, conf, chunkPath, IntWritable.class, LongWritable.class);
-          currentChunkSize = 0;
-        }
-
-        int fieldSize = SEQUENCEFILE_BYTE_OVERHEAD + Integer.SIZE / 8 + Long.SIZE / 8;
-        currentChunkSize += fieldSize;
-        if (key.get() >= 0) {
-          freqWriter.append(key, value);
-        } else if (key.get() == -1) {
-          vectorCount = value.get();
-        }
-        featureCount = Math.max(key.get(), featureCount);
+    Path filesPattern = new Path(featureCountPath, OUTPUT_FILES_PATTERN);
+    for (Pair<IntWritable,LongWritable> record :
+         new SequenceFileDirIterable<IntWritable,LongWritable>(filesPattern, PathType.GLOB, null, null, true, conf)) {
+
+      if (currentChunkSize > chunkSizeLimit) {
+        freqWriter.close();
+        chunkIndex++;
+
+        chunkPath = new Path(dictionaryPathBase, FREQUENCY_FILE + chunkIndex);
+        chunkPaths.add(chunkPath);
 
+        freqWriter = new SequenceFile.Writer(fs, conf, chunkPath, IntWritable.class, LongWritable.class);
+        currentChunkSize = 0;
       }
+
+      int fieldSize = SEQUENCEFILE_BYTE_OVERHEAD + Integer.SIZE / 8 + Long.SIZE / 8;
+      currentChunkSize += fieldSize;
+      IntWritable key = record.getFirst();
+      LongWritable value = record.getSecond();
+      if (key.get() >= 0) {
+        freqWriter.append(key, value);
+      } else if (key.get() == -1) {
+        vectorCount = value.get();
+      }
+      featureCount = Math.max(key.get(), featureCount);
+
     }
-    featureCount++;
+  featureCount++;
     freqWriter.close();
     Long[] counts = {featureCount, vectorCount};
     return new Pair<Long[], List<Path>>(counts, chunkPaths);
@@ -309,7 +305,7 @@ public final class TFIDFConverter {
     job.setReducerClass(TFIDFPartialVectorReducer.class);
     job.setOutputFormatClass(SequenceFileOutputFormat.class);
 
-    HadoopUtil.overwriteOutput(output);
+    HadoopUtil.delete(conf, output);
 
     job.waitForCompletion(true);
   }
@@ -343,7 +339,7 @@ public final class TFIDFConverter {
     job.setReducerClass(TermDocumentCountReducer.class);
     job.setOutputFormatClass(SequenceFileOutputFormat.class);
 
-    HadoopUtil.overwriteOutput(output);
+    HadoopUtil.delete(conf, output);
 
     job.waitForCompletion(true);
   }

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFPartialVectorReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFPartialVectorReducer.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFPartialVectorReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/tfidf/TFIDFPartialVectorReducer.java Thu Mar 31 09:25:25 2011
@@ -24,13 +24,13 @@ import java.util.Iterator;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.mahout.common.Pair;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileIterable;
 import org.apache.mahout.math.NamedVector;
 import org.apache.mahout.math.RandomAccessSparseVector;
 import org.apache.mahout.math.SequentialAccessSparseVector;
@@ -114,14 +114,10 @@ public class TFIDFPartialVectorReducer e
     namedVector = conf.getBoolean(PartialVectorMerger.NAMED_VECTOR, false);
 
     Path dictionaryFile = new Path(localFiles[0].getPath());
-    FileSystem fs = dictionaryFile.getFileSystem(conf);
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, dictionaryFile, conf);
-    IntWritable key = new IntWritable();
-    LongWritable value = new LongWritable();
-
     // key is feature, value is the document frequency
-    while (reader.next(key, value)) {
-      dictionary.put(key.get(), value.get());
+    for (Pair<IntWritable,LongWritable> record :
+         new SequenceFileIterable<IntWritable,LongWritable>(dictionaryFile, true, conf)) {
+      dictionary.put(record.getFirst().get(), record.getSecond().get());
     }
   }
 

Modified: mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/als/ParallelALSFactorizationJobTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/als/ParallelALSFactorizationJobTest.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/als/ParallelALSFactorizationJobTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/als/ParallelALSFactorizationJobTest.java Thu Mar 31 09:25:25 2011
@@ -18,7 +18,6 @@
 package org.apache.mahout.cf.taste.hadoop.als;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.LongWritable;
@@ -28,6 +27,7 @@ import org.apache.hadoop.mapreduce.Reduc
 import org.apache.mahout.cf.taste.hadoop.TasteHadoopUtils;
 import org.apache.mahout.cf.taste.impl.TasteTestCase;
 import org.apache.mahout.cf.taste.impl.common.FullRunningAverage;
+import org.apache.mahout.cf.taste.impl.common.RunningAverage;
 import org.apache.mahout.math.DenseVector;
 import org.apache.mahout.math.Matrix;
 import org.apache.mahout.math.MatrixSlice;
@@ -81,12 +81,12 @@ public class ParallelALSFactorizationJob
   public void initializeMReducer() throws Exception {
     Reducer<VarLongWritable,FloatWritable,VarIntWritable,FeatureVectorWithRatingWritable>.Context ctx =
         EasyMock.createMock(Reducer.Context.class);
-    ctx.write(EasyMock.eq(new VarIntWritable(TasteHadoopUtils.idToIndex(123L))), matchInitializedFeatureVector(3d, 3));
+    ctx.write(EasyMock.eq(new VarIntWritable(TasteHadoopUtils.idToIndex(123L))), matchInitializedFeatureVector(3.0, 3));
     EasyMock.replay(ctx);
 
     ParallelALSFactorizationJob.InitializeMReducer reducer = new ParallelALSFactorizationJob.InitializeMReducer();
     setField(reducer, "numFeatures", 3);
-    reducer.reduce(new VarLongWritable(123L), Arrays.asList(new FloatWritable(4f), new FloatWritable(2f)), ctx);
+    reducer.reduce(new VarLongWritable(123L), Arrays.asList(new FloatWritable(4.0f), new FloatWritable(2.0f)), ctx);
     EasyMock.verify(ctx);
   }
 
@@ -215,10 +215,10 @@ public class ParallelALSFactorizationJob
 
     Double na = Double.NaN;
     Matrix preferences = new SparseRowMatrix(new int[] { 4, 4 }, new Vector[] {
-        new DenseVector(new double[] { 5d, 5d, 2d, na }),
-        new DenseVector(new double[] { 2d, na, 3d, 5d }),
-        new DenseVector(new double[] { na, 5d, na, 3d }),
-        new DenseVector(new double[] { 3d, na, na, 5d }) });
+        new DenseVector(new double[] {5.0, 5.0, 2.0,  na }),
+        new DenseVector(new double[] {2.0,  na, 3.0, 5.0 }),
+        new DenseVector(new double[] { na, 5.0,  na, 3.0 }),
+        new DenseVector(new double[] {3.0,  na,  na, 5.0 }) });
 
     StringBuilder prefsAsText = new StringBuilder();
     String separator = "";
@@ -229,7 +229,7 @@ public class ParallelALSFactorizationJob
       while (elementIterator.hasNext()) {
         Vector.Element e = elementIterator.next();
         if (!Double.isNaN(e.get())) {
-          prefsAsText.append(separator).append(slice.index()).append(",").append(e.index()).append(",").append(e.get());
+          prefsAsText.append(separator).append(slice.index()).append(',').append(e.index()).append(',').append(e.get());
           separator = "\n";
         }
       }
@@ -244,23 +244,19 @@ public class ParallelALSFactorizationJob
     conf.set("mapred.output.dir", outputDir.getAbsolutePath());
     conf.setBoolean("mapred.output.compress", false);
 
+    alsFactorization.setConf(conf);
     int numFeatures = 3;
     int numIterations = 5;
     double lambda = 0.065;
-
-    alsFactorization.setConf(conf);
     alsFactorization.run(new String[] { "--tempDir", tmpDir.getAbsolutePath(), "--lambda", String.valueOf(lambda),
         "--numFeatures", String.valueOf(numFeatures), "--numIterations", String.valueOf(numIterations) });
 
-    Path inputPath = new Path(inputFile.getAbsolutePath());
-    FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
-
-    Matrix u = MathHelper.readEntries(fs, conf, new Path(outputDir.getAbsolutePath(), "U/part-r-00000"),
+    Matrix u = MathHelper.readEntries(conf, new Path(outputDir.getAbsolutePath(), "U/part-r-00000"),
         preferences.numRows(), numFeatures);
-    Matrix m = MathHelper.readEntries(fs, conf, new Path(outputDir.getAbsolutePath(), "M/part-r-00000"),
+    Matrix m = MathHelper.readEntries(conf, new Path(outputDir.getAbsolutePath(), "M/part-r-00000"),
       preferences.numCols(), numFeatures);
 
-    FullRunningAverage avg = new FullRunningAverage();
+    RunningAverage avg = new FullRunningAverage();
     sliceIterator = preferences.iterateAll();
     while (sliceIterator.hasNext()) {
       MatrixSlice slice = sliceIterator.next();
@@ -273,7 +269,7 @@ public class ParallelALSFactorizationJob
           double err = pref - estimate;
           avg.addDatum(err * err);
           logger.info("Comparing preference of user [" + slice.index() + "] towards item [" + e.index() + "], " +
-              "was [" + pref + "] estimate is [" + estimate + "]");
+              "was [" + pref + "] estimate is [" + estimate + ']');
         }
       }
     }

Modified: mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java Thu Mar 31 09:25:25 2011
@@ -41,7 +41,7 @@ import org.apache.mahout.cf.taste.impl.T
 import org.apache.mahout.cf.taste.impl.common.FastIDSet;
 import org.apache.mahout.cf.taste.impl.recommender.GenericRecommendedItem;
 import org.apache.mahout.cf.taste.recommender.RecommendedItem;
-import org.apache.mahout.common.FileLineIterable;
+import org.apache.mahout.common.iterator.FileLineIterable;
 import org.apache.mahout.math.RandomAccessSparseVector;
 import org.apache.mahout.math.VarIntWritable;
 import org.apache.mahout.math.VarLongWritable;

Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java Thu Mar 31 09:25:25 2011
@@ -107,8 +107,7 @@ public final class BayesClassifierSelfTe
     TestClassifier.classifyParallel(params);
     Configuration conf = new Configuration();
     Path outputFiles = getTestTempFilePath("bayesinput-output/part*");
-    FileSystem fs = FileSystem.get(outputFiles.toUri(), conf);
-    matrix = BayesClassifierDriver.readResult(fs, outputFiles, conf, params).getConfusionMatrix();
+    matrix = BayesClassifierDriver.readResult(outputFiles, conf, params).getConfusionMatrix();
     for (int i = 0; i < 3; i++) {
       for (int j = 0; j < 3; j++) {
         assertEquals(i == j ? 4 : 0, matrix[i][j]);
@@ -158,8 +157,7 @@ public final class BayesClassifierSelfTe
     TestClassifier.classifyParallel(params);
     Configuration conf = new Configuration();
     Path outputFiles = getTestTempFilePath("bayesinput-output/part*");
-    FileSystem fs = FileSystem.get(outputFiles.toUri(), conf);
-    matrix = BayesClassifierDriver.readResult(fs, outputFiles, conf, params).getConfusionMatrix();
+    matrix = BayesClassifierDriver.readResult(outputFiles, conf, params).getConfusionMatrix();
     for (int i = 0; i < 3; i++) {
       for (int j = 0; j < 3; j++) {
         assertEquals(i == j ? 4 : 0, matrix[i][j]);

Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java Thu Mar 31 09:25:25 2011
@@ -27,8 +27,8 @@ import java.util.Iterator;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.mahout.classifier.BayesFileFormatter;
-import org.apache.mahout.common.FileLineIterable;
-import org.apache.mahout.common.FileLineIterator;
+import org.apache.mahout.common.iterator.FileLineIterable;
+import org.apache.mahout.common.iterator.FileLineIterator;
 import org.apache.mahout.common.MahoutTestCase;
 import org.junit.Before;
 import org.junit.Test;

Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java Thu Mar 31 09:25:25 2011
@@ -35,16 +35,17 @@ import org.apache.hadoop.io.WritableComp
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.util.ToolRunner;
-import org.apache.mahout.clustering.AbstractCluster;
 import org.apache.mahout.clustering.ClusteringTestUtils;
 import org.apache.mahout.clustering.WeightedVectorWritable;
 import org.apache.mahout.common.DummyRecordWriter;
+import org.apache.mahout.common.HadoopUtil;
 import org.apache.mahout.common.MahoutTestCase;
 import org.apache.mahout.common.commandline.DefaultOptionCreator;
 import org.apache.mahout.common.distance.DistanceMeasure;
 import org.apache.mahout.common.distance.EuclideanDistanceMeasure;
 import org.apache.mahout.common.distance.ManhattanDistanceMeasure;
 import org.apache.mahout.common.distance.UserDefinedDistanceMeasure;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileValueIterable;
 import org.apache.mahout.math.RandomAccessSparseVector;
 import org.apache.mahout.math.Vector;
 import org.apache.mahout.math.VectorWritable;
@@ -435,27 +436,15 @@ public final class TestCanopyCreation ex
 
     // verify output from sequence file
     Path path = new Path(output, "clusters-0/part-r-00000");
-    FileSystem fs = FileSystem.get(path.toUri(), config);
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, config);
-    Writable key = new Text();
-    Canopy canopy = new Canopy();
     int ix = 0;
-    while (reader.next(key, canopy)) {
-      assertEquals("Center [" + ix + ']', manhattanCentroids.get(ix), canopy.getCenter());
+    for (Canopy value : new SequenceFileValueIterable<Canopy>(path, true, config)) {
+      assertEquals("Center [" + ix + ']', manhattanCentroids.get(ix), value.getCenter());
       ix++;
     }
-    reader.close();
+
     path = new Path(output, "clusteredPoints/part-m-0");
-    reader = new SequenceFile.Reader(fs, path, config);
-    int count = 0;
-    Writable clusterId = new IntWritable(0);
-    WeightedVectorWritable vector = new WeightedVectorWritable();
-    while (reader.next(clusterId, vector)) {
-      count++;
-      System.out.println("Txt: " + clusterId + " Vec: " + AbstractCluster.formatVector(vector.getVector(), null));
-    }
+    long count = HadoopUtil.countRecords(path, config);
     assertEquals("number of points", points.size(), count);
-    reader.close();
   }
 
   /** Story: User can cluster points using sequential execution */
@@ -476,27 +465,16 @@ public final class TestCanopyCreation ex
 
     // verify output from sequence file
     Path path = new Path(output, "clusters-0/part-r-00000");
-    FileSystem fs = FileSystem.get(path.toUri(), config);
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, config);
-    Writable key = new Text();
-    Canopy canopy = new Canopy();
+
     int ix = 0;
-    while (reader.next(key, canopy)) {
-      assertEquals("Center [" + ix + ']', euclideanCentroids.get(ix), canopy.getCenter());
+    for (Canopy value : new SequenceFileValueIterable<Canopy>(path, true, config)) {
+      assertEquals("Center [" + ix + ']', euclideanCentroids.get(ix), value.getCenter());
       ix++;
     }
-    reader.close();
+
     path = new Path(output, "clusteredPoints/part-m-0");
-    reader = new SequenceFile.Reader(fs, path, config);
-    int count = 0;
-    Writable clusterId = new IntWritable(0);
-    WeightedVectorWritable vector = new WeightedVectorWritable();
-    while (reader.next(clusterId, vector)) {
-      count++;
-      System.out.println("Txt: " + clusterId + " Vec: " + AbstractCluster.formatVector(vector.getVector(), null));
-    }
+    long count = HadoopUtil.countRecords(path, config);
     assertEquals("number of points", points.size(), count);
-    reader.close();
   }
 
   /**
@@ -513,16 +491,8 @@ public final class TestCanopyCreation ex
     Path output = getTestTempDirPath("output");
     CanopyDriver.run(conf, getTestTempDirPath("testdata"), output, manhattanDistanceMeasure, 3.1, 2.1, true, false);
     Path path = new Path(output, "clusteredPoints/part-m-00000");
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
-    int count = 0;
-    Writable clusterId = new IntWritable(0);
-    WeightedVectorWritable vector = new WeightedVectorWritable();
-    while (reader.next(clusterId, vector)) {
-      count++;
-      System.out.println("Txt: " + clusterId + " Vec: " + AbstractCluster.formatVector(vector.getVector(), null));
-    }
+    long count = HadoopUtil.countRecords(path, conf);
     assertEquals("number of points", points.size(), count);
-    reader.close();
   }
 
   /**
@@ -544,16 +514,8 @@ public final class TestCanopyCreation ex
         optKey(DefaultOptionCreator.OVERWRITE_OPTION) };
     ToolRunner.run(new Configuration(), new CanopyDriver(), args);
     Path path = new Path(output, "clusteredPoints/part-m-00000");
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
-    int count = 0;
-    Writable canopyId = new IntWritable(0);
-    WeightedVectorWritable vw = new WeightedVectorWritable();
-    while (reader.next(canopyId, vw)) {
-      count++;
-      System.out.println("Txt: " + canopyId.toString() + " Vec: " + AbstractCluster.formatVector(vw.getVector(), null));
-    }
+    long count = HadoopUtil.countRecords(path, conf);
     assertEquals("number of points", points.size(), count);
-    reader.close();
   }
 
   /** Story: Clustering algorithm must support arbitrary user defined distance measure */

Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java Thu Mar 31 09:25:25 2011
@@ -31,7 +31,6 @@ import org.apache.hadoop.io.DataOutputBu
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
@@ -42,6 +41,7 @@ import org.apache.mahout.clustering.Clus
 import org.apache.mahout.clustering.WeightedVectorWritable;
 import org.apache.mahout.clustering.kmeans.TestKmeansClustering;
 import org.apache.mahout.common.DummyRecordWriter;
+import org.apache.mahout.common.HadoopUtil;
 import org.apache.mahout.common.MahoutTestCase;
 import org.apache.mahout.common.commandline.DefaultOptionCreator;
 import org.apache.mahout.common.distance.DistanceMeasure;
@@ -242,13 +242,8 @@ public final class TestFuzzyKmeansCluste
           DefaultOptionCreator.SEQUENTIAL_METHOD
       };
       new FuzzyKMeansDriver().run(args);
-      SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(output, "clusteredPoints/part-m-0"), conf);
-      Writable key = new IntWritable();
-      Writable out = new WeightedVectorWritable();
-      while (reader.next(key, out)) {
-        // make sure we can read all the clusters
-      }
-      reader.close();
+      long count = HadoopUtil.countRecords(new Path(output, "clusteredPoints/part-m-0"), conf);
+      assertTrue(count > 0);
     }
 
   }
@@ -318,13 +313,8 @@ public final class TestFuzzyKmeansCluste
           optKey(DefaultOptionCreator.OVERWRITE_OPTION)
       };
       ToolRunner.run(new Configuration(), new FuzzyKMeansDriver(), args);
-      SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(output, "clusteredPoints/part-m-00000"), conf);
-      Writable key = new IntWritable();
-      Writable out = new WeightedVectorWritable();
-      while (reader.next(key, out)) {
-        // make sure we can read all the clusters
-      }
-      reader.close();
+      long count = HadoopUtil.countRecords(new Path(output, "clusteredPoints/part-m-00000"), conf);
+      assertTrue(count > 0);
     }
 
   }
@@ -612,7 +602,8 @@ public final class TestFuzzyKmeansCluste
       }
 
       FuzzyKMeansClusterMapper clusterMapper = new FuzzyKMeansClusterMapper();
-      DummyRecordWriter<IntWritable, WeightedVectorWritable> clusterWriter = new DummyRecordWriter<IntWritable, WeightedVectorWritable>();
+      DummyRecordWriter<IntWritable, WeightedVectorWritable> clusterWriter =
+          new DummyRecordWriter<IntWritable, WeightedVectorWritable>();
       Mapper<WritableComparable<?>, VectorWritable, IntWritable, WeightedVectorWritable>.Context clusterContext =
           DummyRecordWriter.build(clusterMapper, conf, clusterWriter);
       clusterMapper.setup(reducerClusters, conf);
@@ -640,9 +631,8 @@ public final class TestFuzzyKmeansCluste
                                                                             2,
                                                                             1);
 
-      computeCluster(pointsVectors, clusters.get(clusters.size() - 1), new FuzzyKMeansClusterer(new EuclideanDistanceMeasure(),
-                                                                                                0.001,
-                                                                                                2), refClusters);
+      computeCluster(pointsVectors, clusters.get(clusters.size() - 1),
+                     new FuzzyKMeansClusterer(new EuclideanDistanceMeasure(), 0.001, 2), refClusters);
 
       // Now compare the clustermapper results with reference implementation
       assertEquals("mapper and reference sizes", refClusters.size(), clusterWriter.getKeys().size());

Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java Thu Mar 31 09:25:25 2011
@@ -42,10 +42,12 @@ import org.apache.mahout.clustering.cano
 import org.apache.mahout.common.DummyOutputCollector;
 import org.apache.mahout.common.DummyRecordWriter;
 import org.apache.mahout.common.MahoutTestCase;
+import org.apache.mahout.common.Pair;
 import org.apache.mahout.common.commandline.DefaultOptionCreator;
 import org.apache.mahout.common.distance.DistanceMeasure;
 import org.apache.mahout.common.distance.EuclideanDistanceMeasure;
 import org.apache.mahout.common.distance.ManhattanDistanceMeasure;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileIterable;
 import org.apache.mahout.math.DenseVector;
 import org.apache.mahout.math.RandomAccessSparseVector;
 import org.apache.mahout.math.SequentialAccessSparseVector;
@@ -395,20 +397,15 @@ public final class TestKmeansClustering 
 
       // now compare the expected clusters with actual
       Path clusteredPointsPath = new Path(outputPath, "clusteredPoints");
-      SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(clusteredPointsPath, "part-m-0"), conf);
       int[] expect = EXPECTED_NUM_POINTS[k];
       DummyOutputCollector<IntWritable, WeightedVectorWritable> collector =
           new DummyOutputCollector<IntWritable, WeightedVectorWritable>();
-      // The key is the clusterId
-      IntWritable clusterId = new IntWritable(0);
-      // The value is the weighted vector
-      WeightedVectorWritable value = new WeightedVectorWritable();
-      while (reader.next(clusterId, value)) {
-        collector.collect(clusterId, value);
-        clusterId = new IntWritable(0);
-        value = new WeightedVectorWritable();
+      // The key is the clusterId, the value is the weighted vector
+      for (Pair<IntWritable,WeightedVectorWritable> record :
+           new SequenceFileIterable<IntWritable,WeightedVectorWritable>(
+               new Path(clusteredPointsPath, "part-m-0"), conf)) {
+        collector.collect(record.getFirst(), record.getSecond());
       }
-      reader.close();
       assertEquals("clusters[" + k + ']', expect.length, collector.getKeys().size());
     }
   }
@@ -453,20 +450,15 @@ public final class TestKmeansClustering 
       // now compare the expected clusters with actual
       Path clusteredPointsPath = new Path(outputPath, "clusteredPoints");
       // assertEquals("output dir files?", 4, outFiles.length);
-      SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(clusteredPointsPath, "part-m-00000"), conf);
       int[] expect = EXPECTED_NUM_POINTS[k];
       DummyOutputCollector<IntWritable, WeightedVectorWritable> collector =
           new DummyOutputCollector<IntWritable, WeightedVectorWritable>();
-      // The key is the clusterId
-      IntWritable clusterId = new IntWritable(0);
-      // The value is the weighted vector
-      WeightedVectorWritable value = new WeightedVectorWritable();
-      while (reader.next(clusterId, value)) {
-        collector.collect(clusterId, value);
-        clusterId = new IntWritable(0);
-        value = new WeightedVectorWritable();
+      // The key is the clusterId, the value is the weighted vector
+      for (Pair<IntWritable,WeightedVectorWritable> record :
+           new SequenceFileIterable<IntWritable,WeightedVectorWritable>(
+               new Path(clusteredPointsPath, "part-m-00000"), conf)) {
+        collector.collect(record.getFirst(), record.getSecond());
       }
-      reader.close();
       if (k == 2) {
         // cluster 3 is empty so won't appear in output
         assertEquals("clusters[" + k + ']', expect.length - 1, collector.getKeys().size());
@@ -504,19 +496,13 @@ public final class TestKmeansClustering 
     Path clusteredPointsPath = new Path(outputPath, "clusteredPoints");
     DummyOutputCollector<IntWritable, WeightedVectorWritable> collector =
         new DummyOutputCollector<IntWritable, WeightedVectorWritable>();
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(clusteredPointsPath, "part-m-00000"), conf);
-
-    // The key is the clusterId
-    IntWritable clusterId = new IntWritable(0);
-    // The value is the vector
-    WeightedVectorWritable value = new WeightedVectorWritable();
-    while (reader.next(clusterId, value)) {
-      collector.collect(clusterId, value);
-      clusterId = new IntWritable(0);
-      value = new WeightedVectorWritable();
 
+    // The key is the clusterId, the value is the weighted vector
+    for (Pair<IntWritable,WeightedVectorWritable> record :
+         new SequenceFileIterable<IntWritable,WeightedVectorWritable>(
+             new Path(clusteredPointsPath, "part-m-00000"), conf)) {
+      collector.collect(record.getFirst(), record.getSecond());
     }
-    reader.close();
 
     assertEquals("num points[0]", 4, collector.getValue(new IntWritable(0)).size());
     assertEquals("num points[1]", 5, collector.getValue(new IntWritable(1)).size());

Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestRandomSeedGenerator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestRandomSeedGenerator.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestRandomSeedGenerator.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestRandomSeedGenerator.java Thu Mar 31 09:25:25 2011
@@ -25,13 +25,12 @@ import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.mahout.clustering.AbstractCluster;
 import org.apache.mahout.clustering.ClusteringTestUtils;
 import org.apache.mahout.common.MahoutTestCase;
 import org.apache.mahout.common.distance.ManhattanDistanceMeasure;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileValueIterable;
 import org.apache.mahout.math.RandomAccessSparseVector;
 import org.apache.mahout.math.Vector;
 import org.apache.mahout.math.VectorWritable;
@@ -75,14 +74,11 @@ public final class TestRandomSeedGenerat
     ClusteringTestUtils.writePointsToFile(points, input, fs, conf);
     
     RandomSeedGenerator.buildRandom(input, output, 4, new ManhattanDistanceMeasure());
-    
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path(output, "part-randomSeed"), conf);
-    Writable key = reader.getKeyClass().asSubclass(Writable.class).newInstance();
-    AbstractCluster value = (AbstractCluster) reader.getValueClass().newInstance();
-    
+
     int clusterCount = 0;
     Collection<Integer> set = new HashSet<Integer>();
-    while (reader.next(key, value)) {
+    for (AbstractCluster value :
+         new SequenceFileValueIterable<AbstractCluster>(new Path(output, "part-randomSeed"), true, conf)) {
       clusterCount++;
       int id = value.getId();
       assertTrue(set.add(id)); // validate unique id's

Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java Thu Mar 31 09:25:25 2011
@@ -27,15 +27,14 @@ import java.util.Map;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.mahout.clustering.ClusteringTestUtils;
 import org.apache.mahout.common.DummyRecordWriter;
+import org.apache.mahout.common.HadoopUtil;
 import org.apache.mahout.common.MahoutTestCase;
 import org.apache.mahout.common.commandline.DefaultOptionCreator;
 import org.apache.mahout.common.distance.DistanceMeasure;
@@ -151,7 +150,8 @@ public final class TestMeanShift extends
   @Test
   public void testClustererReferenceImplementation() {
     Iterable<Vector> points = new ArrayList<Vector>(Arrays.asList(raw));
-    List<MeanShiftCanopy> canopies = MeanShiftCanopyClusterer.clusterPoints(points, euclideanDistanceMeasure, 0.5, 4, 1, 10);
+    List<MeanShiftCanopy> canopies =
+        MeanShiftCanopyClusterer.clusterPoints(points, euclideanDistanceMeasure, 0.5, 4, 1, 10);
     printCanopies(canopies);
     printImage(canopies);
   }
@@ -181,9 +181,8 @@ public final class TestMeanShift extends
     // map the data
     MeanShiftCanopyMapper mapper = new MeanShiftCanopyMapper();
     DummyRecordWriter<Text, MeanShiftCanopy> mapWriter = new DummyRecordWriter<Text, MeanShiftCanopy>();
-    Mapper<WritableComparable<?>, MeanShiftCanopy, Text, MeanShiftCanopy>.Context mapContext = DummyRecordWriter.build(mapper,
-                                                                                                                       conf,
-                                                                                                                       mapWriter);
+    Mapper<WritableComparable<?>, MeanShiftCanopy, Text, MeanShiftCanopy>.Context mapContext =
+        DummyRecordWriter.build(mapper, conf, mapWriter);
     mapper.setup(mapContext);
     for (MeanShiftCanopy canopy : canopies) {
       mapper.map(new Text(), canopy, mapContext);
@@ -253,9 +252,8 @@ public final class TestMeanShift extends
 
     MeanShiftCanopyMapper mapper = new MeanShiftCanopyMapper();
     DummyRecordWriter<Text, MeanShiftCanopy> mapWriter = new DummyRecordWriter<Text, MeanShiftCanopy>();
-    Mapper<WritableComparable<?>, MeanShiftCanopy, Text, MeanShiftCanopy>.Context mapContext = DummyRecordWriter.build(mapper,
-                                                                                                                       conf,
-                                                                                                                       mapWriter);
+    Mapper<WritableComparable<?>, MeanShiftCanopy, Text, MeanShiftCanopy>.Context mapContext =
+        DummyRecordWriter.build(mapper, conf, mapWriter);
     mapper.setup(mapContext);
 
     // map the data
@@ -268,11 +266,8 @@ public final class TestMeanShift extends
     // now reduce the mapper output
     MeanShiftCanopyReducer reducer = new MeanShiftCanopyReducer();
     DummyRecordWriter<Text, MeanShiftCanopy> reduceWriter = new DummyRecordWriter<Text, MeanShiftCanopy>();
-    Reducer<Text, MeanShiftCanopy, Text, MeanShiftCanopy>.Context reduceContext = DummyRecordWriter.build(reducer,
-                                                                                                          conf,
-                                                                                                          reduceWriter,
-                                                                                                          Text.class,
-                                                                                                          MeanShiftCanopy.class);
+    Reducer<Text, MeanShiftCanopy, Text, MeanShiftCanopy>.Context reduceContext =
+        DummyRecordWriter.build(reducer, conf, reduceWriter, Text.class, MeanShiftCanopy.class);
     reducer.setup(reduceContext);
     reducer.reduce(new Text("canopy"), mapWriter.getValue(new Text("canopy")), reduceContext);
     reducer.cleanup(reduceContext);
@@ -330,14 +325,7 @@ public final class TestMeanShift extends
         optKey(DefaultOptionCreator.OVERWRITE_OPTION) };
     ToolRunner.run(conf, new MeanShiftCanopyDriver(), args);
     Path outPart = new Path(output, "clusters-3/part-r-00000");
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, outPart, conf);
-    Writable key = new Text();
-    Writable value = new MeanShiftCanopy();
-    int count = 0;
-    while (reader.next(key, value)) {
-      count++;
-    }
-    reader.close();
+    long count = HadoopUtil.countRecords(outPart, conf);
     assertEquals("count", 3, count);
   }
 
@@ -369,14 +357,7 @@ public final class TestMeanShift extends
         DefaultOptionCreator.SEQUENTIAL_METHOD };
     ToolRunner.run(new Configuration(), new MeanShiftCanopyDriver(), args);
     Path outPart = new Path(output, "clusters-4/part-r-00000");
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, outPart, conf);
-    Writable key = new Text();
-    Writable value = new MeanShiftCanopy();
-    int count = 0;
-    while (reader.next(key, value)) {
-      count++;
-    }
-    reader.close();
+    long count = HadoopUtil.countRecords(outPart, conf);
     assertEquals("count", 5, count);
   }
 }

Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java Thu Mar 31 09:25:25 2011
@@ -25,8 +25,10 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.mahout.clustering.minhash.HashFactory.HashType;
 import org.apache.mahout.common.MahoutTestCase;
+import org.apache.mahout.common.Pair;
 import org.apache.mahout.common.commandline.DefaultOptionCreator;
 import org.apache.mahout.common.commandline.MinhashOptionCreator;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileIterable;
 import org.apache.mahout.math.SequentialAccessSparseVector;
 import org.apache.mahout.math.Vector;
 import org.apache.mahout.math.VectorWritable;
@@ -45,8 +47,7 @@ public class TestMinHashClustering exten
                                               {4, 7, 8, 9, 6, 1}, {5, 8, 10, 4, 1}, {6, 17, 14, 15},
                                               {8, 9, 11, 6, 12, 1, 7}, {10, 13, 9, 7, 4, 6, 3},
                                               {3, 5, 7, 9, 2, 11}, {13, 7, 6, 8, 5}};
-  
-  private FileSystem fs;
+
   private Path input;
   private Path output;
   
@@ -64,13 +65,12 @@ public class TestMinHashClustering exten
   public void setUp() throws Exception {
     super.setUp();
     Configuration conf = new Configuration();
-    fs = FileSystem.get(conf);
+    FileSystem fs = FileSystem.get(conf);
     List<VectorWritable> points = getPointsWritable(REFERENCE);
     input = getTestTempDirPath("points");
     output = new Path(getTestTempDirPath(), "output");
     Path pointFile = new Path(input, "file1");
-    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, pointFile, Text.class,
-        VectorWritable.class);
+    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, pointFile, Text.class, VectorWritable.class);
     int id = 0;
     for (VectorWritable point : points) {
       writer.append(new Text("Id-" + id++), point);
@@ -123,22 +123,21 @@ public class TestMinHashClustering exten
     }
   }
   
-  private void verify(Path output, double simThreshold, String msg) throws Exception {
+  private static void verify(Path output, double simThreshold, String msg) {
     Configuration conf = new Configuration();
     Path outputFile = new Path(output, "part-r-00000");
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, outputFile, conf);
-    Writable clusterId = new Text();
-    VectorWritable point = new VectorWritable();
     List<Vector> clusteredItems = new ArrayList<Vector>();
     String prevClusterId = "";
-    while (reader.next(clusterId, point)) {
+    for (Pair<Writable,VectorWritable> record : new SequenceFileIterable<Writable,VectorWritable>(outputFile, conf)) {
+      Writable clusterId = record.getFirst();
+      VectorWritable point = record.getSecond();
       if (prevClusterId.equals(clusterId.toString())) {
-        clusteredItems.add(point.get().clone());
+        clusteredItems.add(point.get());
       } else {
         runPairwiseSimilarity(clusteredItems, simThreshold, msg);
         clusteredItems.clear();
         prevClusterId = clusterId.toString();
-        clusteredItems.add(point.get().clone());
+        clusteredItems.add(point.get());
       }
     }
     runPairwiseSimilarity(clusteredItems, simThreshold, msg);

Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/spectral/common/TestVectorCache.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/spectral/common/TestVectorCache.java?rev=1087225&r1=1087224&r2=1087225&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/spectral/common/TestVectorCache.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/spectral/common/TestVectorCache.java Thu Mar 31 09:25:25 2011
@@ -28,6 +28,7 @@ import org.apache.hadoop.io.SequenceFile
 import org.apache.hadoop.io.Writable;
 import org.apache.mahout.common.HadoopUtil;
 import org.apache.mahout.common.MahoutTestCase;
+import org.apache.mahout.common.iterator.sequencefile.SequenceFileValueIterator;
 import org.apache.mahout.math.DenseVector;
 import org.apache.mahout.math.Vector;
 import org.apache.mahout.math.VectorWritable;
@@ -40,7 +41,6 @@ public class TestVectorCache extends Mah
   @Test
   public void testSave() throws Exception {
     Configuration conf = new Configuration();
-    FileSystem fs = FileSystem.get(conf);
     Writable key = new IntWritable(0);
     Vector value = new DenseVector(VECTOR);
     Path path = getTestTempDirPath("output");
@@ -49,10 +49,10 @@ public class TestVectorCache extends Mah
     VectorCache.save(key, value, path, conf, true, true);
     
     // can we read it from here?
-    SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
-    VectorWritable old = new VectorWritable();
-    reader.next(key, old);
-    reader.close();
+    SequenceFileValueIterator<VectorWritable> iterator =
+        new SequenceFileValueIterator<VectorWritable>(path, true, conf);
+    VectorWritable old = iterator.next();
+    iterator.close();
     
     // test if the values are identical
     assertEquals("Saved vector is identical to original", old.get(), value);
@@ -70,15 +70,14 @@ public class TestVectorCache extends Mah
     // write the vector
     path = fs.makeQualified(path);
     fs.deleteOnExit(path);
-    HadoopUtil.overwriteOutput(path);
+    HadoopUtil.delete(conf, path);
     DistributedCache.setCacheFiles(new URI[] {path.toUri()}, conf);
-    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, 
-        IntWritable.class, VectorWritable.class);
+    SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class);
     writer.append(key, new VectorWritable(value));
     writer.close();
     
     // load it
-    Vector result = VectorCache.load(key, conf);
+    Vector result = VectorCache.load(conf);
     
     // are they the same?
     assertNotNull("Vector is not null", result);
@@ -90,14 +89,13 @@ public class TestVectorCache extends Mah
     Configuration conf = new Configuration();
     Vector v = new DenseVector(VECTOR);
     Path toSave = getTestTempDirPath("output");
-    IntWritable key = new IntWritable(0);
+    Writable key = new IntWritable(0);
     
     // save it
     VectorCache.save(key, v, toSave, conf);
     
     // now, load it back
-    key = new IntWritable(0);
-    Vector v2 = VectorCache.load(key, conf);
+    Vector v2 = VectorCache.load(conf);
     
     // are they the same?
     assertNotNull("Vector is not null", v2);