You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2009/09/09 17:41:32 UTC
svn commit: r813003 - in /lucene/mahout/trunk:
core/src/main/java/org/apache/mahout/cf/taste/impl/common/
core/src/main/java/org/apache/mahout/cf/taste/impl/model/
core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/
core/src/main/java/org/ap...
Author: srowen
Date: Wed Sep 9 15:41:31 2009
New Revision: 813003
URL: http://svn.apache.org/viewvc?rev=813003&view=rev
Log:
MAHOUT-175
Added:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/FileLineIterable.java
- copied, changed from r812267, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterable.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/FileLineIterator.java
- copied, changed from r812267, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterator.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/IOUtils.java
- copied, changed from r812267, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/IOUtils.java
Removed:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterable.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterator.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/IOUtils.java
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/AbstractJDBCIDMigrator.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/GenericJDBCDataModel.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/jdbc/AbstractJDBCDiffStorage.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/jdbc/AbstractJDBCItemSimilarity.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/cbayes/CBayesThetaNormalizerReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/common/BayesTfIdfReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/common/BayesWeightSummerReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansUtil.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAPrintTopics.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/MahoutEvaluatorTest.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/bookcrossing/BookCrossingDataModel.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/grouplens/GroupLensDataModel.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/jester/JesterDataModel.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/NetflixDataModel.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/NetflixFileDataModel.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/TransposeToByUser.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreatorDriver.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/dirichlet/DisplayASNOutputState.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/dirichlet/DisplayOutputState.java
lucene/mahout/trunk/examples/src/test/java/org/apache/mahout/ga/watchmaker/cd/FileInfosDatasetTest.java
lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java
lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/AbstractJDBCIDMigrator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/AbstractJDBCIDMigrator.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/AbstractJDBCIDMigrator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/AbstractJDBCIDMigrator.java Wed Sep 9 15:41:31 2009
@@ -18,7 +18,7 @@
package org.apache.mahout.cf.taste.impl.model;
import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.impl.common.IOUtils;
+import org.apache.mahout.common.IOUtils;
import javax.sql.DataSource;
import java.sql.Connection;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java Wed Sep 9 15:41:31 2009
@@ -21,7 +21,7 @@
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
import org.apache.mahout.cf.taste.impl.common.FastIDSet;
-import org.apache.mahout.cf.taste.impl.common.FileLineIterator;
+import org.apache.mahout.common.FileLineIterator;
import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
import org.apache.mahout.cf.taste.impl.model.GenericBooleanPrefDataModel;
import org.apache.mahout.cf.taste.impl.model.GenericDataModel;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java Wed Sep 9 15:41:31 2009
@@ -18,7 +18,7 @@
package org.apache.mahout.cf.taste.impl.model.jdbc;
import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.impl.common.IOUtils;
+import org.apache.mahout.common.IOUtils;
import org.apache.mahout.cf.taste.impl.model.BooleanPreference;
import org.apache.mahout.cf.taste.model.Preference;
import org.slf4j.Logger;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java Wed Sep 9 15:41:31 2009
@@ -23,7 +23,7 @@
import org.apache.mahout.cf.taste.impl.common.Cache;
import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
import org.apache.mahout.cf.taste.impl.common.FastIDSet;
-import org.apache.mahout.cf.taste.impl.common.IOUtils;
+import org.apache.mahout.common.IOUtils;
import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
import org.apache.mahout.cf.taste.impl.common.Retriever;
import org.apache.mahout.cf.taste.impl.common.SkippingIterator;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/GenericJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/GenericJDBCDataModel.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/GenericJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/GenericJDBCDataModel.java Wed Sep 9 15:41:31 2009
@@ -18,7 +18,7 @@
package org.apache.mahout.cf.taste.impl.model.jdbc;
import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.impl.common.IOUtils;
+import org.apache.mahout.common.IOUtils;
import java.io.File;
import java.io.FileInputStream;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/jdbc/AbstractJDBCDiffStorage.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/jdbc/AbstractJDBCDiffStorage.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/jdbc/AbstractJDBCDiffStorage.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/jdbc/AbstractJDBCDiffStorage.java Wed Sep 9 15:41:31 2009
@@ -20,7 +20,7 @@
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.impl.common.FastIDSet;
-import org.apache.mahout.cf.taste.impl.common.IOUtils;
+import org.apache.mahout.common.IOUtils;
import org.apache.mahout.cf.taste.impl.common.RefreshHelper;
import org.apache.mahout.cf.taste.impl.common.RunningAverage;
import org.apache.mahout.cf.taste.impl.common.jdbc.AbstractJDBCComponent;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/jdbc/AbstractJDBCItemSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/jdbc/AbstractJDBCItemSimilarity.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/jdbc/AbstractJDBCItemSimilarity.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/jdbc/AbstractJDBCItemSimilarity.java Wed Sep 9 15:41:31 2009
@@ -20,7 +20,7 @@
import org.apache.mahout.cf.taste.common.NoSuchItemException;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.impl.common.IOUtils;
+import org.apache.mahout.common.IOUtils;
import org.apache.mahout.cf.taste.impl.common.jdbc.AbstractJDBCComponent;
import org.apache.mahout.cf.taste.impl.model.jdbc.ConnectionPoolDataSource;
import org.apache.mahout.cf.taste.similarity.ItemSimilarity;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java Wed Sep 9 15:41:31 2009
@@ -29,10 +29,10 @@
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.mahout.common.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Closeable;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
@@ -76,7 +76,7 @@
inputDir.listFiles(new FileProcessor(label, analyzer, charset, writer));
// listFiles() is called here as a way to recursively visit files, actually
} finally {
- quietClose(writer);
+ IOUtils.quietClose(writer);
}
}
@@ -99,7 +99,7 @@
try {
writeFile(label, analyzer, input, charset, writer);
} finally {
- quietClose(writer);
+ IOUtils.quietClose(writer);
}
}
}
@@ -167,7 +167,7 @@
throw new RuntimeException(e);
} finally {
if (writer == null) {
- quietClose(theWriter);
+ IOUtils.quietClose(theWriter);
}
}
} else {
@@ -203,17 +203,7 @@
writer.write(' ');
}
} finally {
- quietClose(reader);
- }
- }
-
- private static void quietClose(Closeable closeable) {
- if (closeable != null) {
- try {
- closeable.close();
- } catch (IOException ioe) {
- // continue
- }
+ IOUtils.quietClose(reader);
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/cbayes/CBayesThetaNormalizerReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/cbayes/CBayesThetaNormalizerReducer.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/cbayes/CBayesThetaNormalizerReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/cbayes/CBayesThetaNormalizerReducer.java Wed Sep 9 15:41:31 2009
@@ -47,6 +47,7 @@
private HBaseConfiguration HBconf;
boolean useHbase = false;
+
@Override
public void reduce(Text key, Iterator<DoubleWritable> values,
OutputCollector<Text, DoubleWritable> output, Reporter reporter)
@@ -94,8 +95,8 @@
@Override
public void close() throws IOException {
if (useHbase) {
- table.close();
- }
+ table.close();
+ }
super.close();
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/common/BayesTfIdfReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/common/BayesTfIdfReducer.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/common/BayesTfIdfReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/common/BayesTfIdfReducer.java Wed Sep 9 15:41:31 2009
@@ -52,6 +52,7 @@
boolean useHbase = false;
+ @Override
public void reduce(Text key, Iterator<DoubleWritable> values,
OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
@@ -119,8 +120,8 @@
@Override
public void close() throws IOException {
if (useHbase) {
- table.close();
- }
+ table.close();
+ }
super.close();
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/common/BayesWeightSummerReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/common/BayesWeightSummerReducer.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/common/BayesWeightSummerReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/common/BayesWeightSummerReducer.java Wed Sep 9 15:41:31 2009
@@ -110,9 +110,9 @@
@Override
public void close() throws IOException {
- if (useHbase) {
+ if (useHbase) {
table.close();
- }
+ }
super.close();
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyMapper.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyMapper.java Wed Sep 9 15:41:31 2009
@@ -50,11 +50,6 @@
Canopy.configure(job);
}
- /*
- * (non-Javadoc)
- *
- * @see org.apache.hadoop.mapred.MapReduceBase#close()
- */
@Override
public void close() throws IOException {
for (Canopy canopy : canopies) {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansUtil.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansUtil.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansUtil.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansUtil.java Wed Sep 9 15:41:31 2009
@@ -26,6 +26,7 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.mahout.clustering.canopy.Canopy;
+import org.apache.mahout.common.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,9 +101,7 @@
}
}
} finally {
- if (reader != null) {
- reader.close();
- }
+ IOUtils.quietClose(reader);
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAPrintTopics.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAPrintTopics.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAPrintTopics.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAPrintTopics.java Wed Sep 9 15:41:31 2009
@@ -17,9 +17,7 @@
package org.apache.mahout.clustering.lda;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
@@ -45,6 +43,7 @@
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.mahout.utils.CommandLineUtil;
+import org.apache.mahout.common.FileLineIterator;
/**
* Class to print out the top K words for each topic.
@@ -141,22 +140,21 @@
// Reads dictionary in created by the vector Driver in util
private static List<String> readDictionary(File path) throws IOException {
- BufferedReader rdr = new BufferedReader(new FileReader(path));
+ FileLineIterator it = new FileLineIterator(path);
List<String> result = new ArrayList<String>();
// skip 2 lines
- rdr.readLine();
- rdr.readLine();
- String line;
- while ( (line = rdr.readLine()) != null) {
+ it.next();
+ it.next();
+ while (it.hasNext()) {
+ String line = it.next();
String[] parts = TAB_PATTERN.split(line);
String word = parts[0];
int index = Integer.parseInt(parts[2]);
assert index == result.size();
result.add(word);
}
- rdr.close();
return result;
}
Copied: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/FileLineIterable.java (from r812267, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterable.java)
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/FileLineIterable.java?p2=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/FileLineIterable.java&p1=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterable.java&r1=812267&r2=813003&rev=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterable.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/FileLineIterable.java Wed Sep 9 15:41:31 2009
@@ -15,10 +15,11 @@
* limitations under the License.
*/
-package org.apache.mahout.cf.taste.impl.common;
+package org.apache.mahout.common;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Iterator;
@@ -30,23 +31,37 @@
*/
public final class FileLineIterable implements Iterable<String> {
- private final File file;
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
+ private final InputStream is;
private final Charset encoding;
private final boolean skipFirstLine;
/** Creates a {@link FileLineIterable} over a given file, assuming a UTF-8 encoding. */
- public FileLineIterable(File file) {
- this(file, Charset.forName("UTF-8"), false);
+ public FileLineIterable(File file) throws IOException {
+ this(file, UTF8, false);
}
/** Creates a {@link FileLineIterable} over a given file, assuming a UTF-8 encoding. */
- public FileLineIterable(File file, boolean skipFirstLine) {
- this(file, Charset.forName("UTF-8"), skipFirstLine);
+ public FileLineIterable(File file, boolean skipFirstLine) throws IOException {
+ this(file, UTF8, skipFirstLine);
}
/** Creates a {@link FileLineIterable} over a given file, using the given encoding. */
- public FileLineIterable(File file, Charset encoding, boolean skipFirstLine) {
- this.file = file;
+ public FileLineIterable(File file, Charset encoding, boolean skipFirstLine) throws IOException {
+ this(FileLineIterator.getFileInputStream(file), encoding, skipFirstLine);
+ }
+
+ public FileLineIterable(InputStream is) {
+ this(is, UTF8, false);
+ }
+
+ public FileLineIterable(InputStream is, boolean skipFirstLine) {
+ this(is, UTF8, skipFirstLine);
+ }
+
+ public FileLineIterable(InputStream is, Charset encoding, boolean skipFirstLine) {
+ this.is = is;
this.encoding = encoding;
this.skipFirstLine = skipFirstLine;
}
@@ -54,7 +69,7 @@
@Override
public Iterator<String> iterator() {
try {
- return new FileLineIterator(file, encoding, skipFirstLine);
+ return new FileLineIterator(is, encoding, skipFirstLine);
} catch (IOException ioe) {
throw new IllegalStateException(ioe);
}
Copied: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/FileLineIterator.java (from r812267, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterator.java)
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/FileLineIterator.java?p2=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/FileLineIterator.java&p1=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterator.java&r1=812267&r2=813003&rev=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FileLineIterator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/FileLineIterator.java Wed Sep 9 15:41:31 2009
@@ -15,7 +15,10 @@
* limitations under the License.
*/
-package org.apache.mahout.cf.taste.impl.common;
+package org.apache.mahout.common;
+
+import org.apache.mahout.cf.taste.impl.common.SkippingIterator;
+import org.apache.mahout.common.IOUtils;
import java.io.BufferedReader;
import java.io.Closeable;
@@ -38,6 +41,8 @@
*/
public final class FileLineIterator implements SkippingIterator<String>, Closeable {
+ private static final Charset UTF8 = Charset.forName("UTF-8");
+
private final BufferedReader reader;
private String nextLine;
@@ -48,7 +53,7 @@
* @throws IOException if the file cannot be read
*/
public FileLineIterator(File file) throws IOException {
- this(file, Charset.forName("UTF-8"), false);
+ this(file, UTF8, false);
}
/**
@@ -58,7 +63,7 @@
* @throws IOException if the file cannot be read
*/
public FileLineIterator(File file, boolean skipFirstLine) throws IOException {
- this(file, Charset.forName("UTF-8"), skipFirstLine);
+ this(file, UTF8, skipFirstLine);
}
/**
@@ -68,7 +73,18 @@
* @throws IOException if the file cannot be read
*/
public FileLineIterator(File file, Charset encoding, boolean skipFirstLine) throws IOException {
- InputStream is = getFileInputStream(file);
+ this(getFileInputStream(file), encoding, skipFirstLine);
+ }
+
+ public FileLineIterator(InputStream is) throws IOException {
+ this(is, UTF8, false);
+ }
+
+ public FileLineIterator(InputStream is, boolean skipFirstLine) throws IOException {
+ this(is, UTF8, skipFirstLine);
+ }
+
+ public FileLineIterator(InputStream is, Charset encoding, boolean skipFirstLine) throws IOException {
reader = new BufferedReader(new InputStreamReader(is, encoding));
if (skipFirstLine) {
reader.readLine();
@@ -76,7 +92,7 @@
nextLine = reader.readLine();
}
- private static InputStream getFileInputStream(File file) throws IOException {
+ static InputStream getFileInputStream(File file) throws IOException {
InputStream is = new FileInputStream(file);
String name = file.getName();
if (name.endsWith(".gz")) {
Copied: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/IOUtils.java (from r812267, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/IOUtils.java)
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/IOUtils.java?p2=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/IOUtils.java&p1=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/IOUtils.java&r1=812267&r2=813003&rev=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/IOUtils.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/IOUtils.java Wed Sep 9 15:41:31 2009
@@ -15,20 +15,13 @@
* limitations under the License.
*/
-package org.apache.mahout.cf.taste.impl.common;
+package org.apache.mahout.common;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayOutputStream;
import java.io.Closeable;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URL;
-import java.nio.charset.Charset;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -98,37 +91,4 @@
quietClose(connection);
}
- public static void copyStreamToFile(InputStream in, File file) throws IOException {
- OutputStream out = new FileOutputStream(file, false);
- copyInToOut(in, out);
- }
-
- public static String readURL(URL url) throws IOException {
- InputStream in = url.openStream();
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- copyInToOut(in, out);
- return new String(out.toByteArray(), Charset.forName("UTF-8"));
- }
-
- public static File createTempFile(String copySuffixFrom) throws IOException {
- int period = copySuffixFrom.lastIndexOf('.');
- String suffix = period < 0 ? "" : copySuffixFrom.substring(period);
- File tempFile = File.createTempFile("mahout-taste-", suffix);
- tempFile.deleteOnExit();
- return tempFile;
- }
-
- public static void copyInToOut(InputStream in, OutputStream out) throws IOException {
- try {
- int bytesRead;
- byte[] buffer = new byte[65536];
- while ((bytesRead = in.read(buffer)) > 0) {
- out.write(buffer, 0, bytesRead);
- }
- } finally {
- quietClose(in);
- quietClose(out);
- }
- }
-
}
Modified: lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java (original)
+++ lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java Wed Sep 9 15:41:31 2009
@@ -21,13 +21,12 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.mahout.classifier.BayesFileFormatter;
+import org.apache.mahout.common.FileLineIterator;
+import org.apache.mahout.common.FileLineIterable;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.Charset;
@@ -74,14 +73,11 @@
assertEquals("files Size: " + files.length + " is not: " + words.length, files.length, words.length);
for (File file : files) {
//should only be one line in the file, and it should be label label
- BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset));
- String line = reader.readLine();
- if (line != null) {
- line = line.trim();
- }
+ FileLineIterator it = new FileLineIterator(file);
+ String line = it.next().trim();
+ assertFalse(it.hasNext());
String label = "animal" + '\t' + file.getName();
assertEquals(line + ":::: is not equal to " + label + "::::", line, label);
- reader.close();
}
}
@@ -93,15 +89,12 @@
BayesFileFormatter.collapse("animal", analyzer, input, charset, new File(out, "animal"));
files = out.listFiles();
assertEquals("files Size: " + files.length + " is not: " + 1, 1, files.length);
- BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(files[0]), charset));
- String line;
int count = 0;
- while ((line = reader.readLine()) != null) {
+ for (String line : new FileLineIterable(files[0])) {
assertTrue("line does not start with label", line.startsWith("animal"));
System.out.println("Line: " + line);
count++;
}
- reader.close();
assertEquals(count + " does not equal: " + words.length, count, words.length);
}
Modified: lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java (original)
+++ lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java Wed Sep 9 15:41:31 2009
@@ -594,9 +594,6 @@
CanopyClusteringJob.runJob("testdata", "output",
ManhattanDistanceMeasure.class.getName(), 3.1, 2.1, SparseVector.class);
//TODO: change
- /*BufferedReader reader = new BufferedReader(new InputStreamReader(
- new FileInputStream("output/clusters/part-00000"), Charset
- .forName("UTF-8")));*/
Path path = new Path("output/clusters/part-00000");
SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
int count = 0;
@@ -628,9 +625,6 @@
// now run the Job
CanopyClusteringJob.runJob("testdata", "output",
EuclideanDistanceMeasure.class.getName(), 3.1, 2.1, SparseVector.class);
- /*BufferedReader reader = new BufferedReader(new InputStreamReader(
- new FileInputStream("output/clusters/part-00000"), Charset
- .forName("UTF-8")));*/
Path path = new Path("output/clusters/part-00000");
SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
int count = 0;
Modified: lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java (original)
+++ lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java Wed Sep 9 15:41:31 2009
@@ -262,9 +262,6 @@
outDir.list();
// assertEquals("output dir files?", 4, outFiles.length);
SequenceFile.Reader reader = new SequenceFile.Reader(fs, new Path("output/points/part-00000"), conf);
- /*BufferedReader reader = new BufferedReader(new InputStreamReader(
- new FileInputStream("output/points/part-00000"), Charset
- .forName("UTF-8")));*/
Text key = new Text();
FuzzyKMeansOutput out = new FuzzyKMeansOutput();
while (reader.next(key, out)) {
Modified: lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java (original)
+++ lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java Wed Sep 9 15:41:31 2009
@@ -36,12 +36,9 @@
import org.apache.mahout.utils.EuclideanDistanceMeasure;
import org.apache.mahout.utils.HadoopUtil;
import org.apache.mahout.utils.ManhattanDistanceMeasure;
+import org.apache.mahout.common.FileLineIterable;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -458,18 +455,13 @@
assertTrue("output dir exists?", outDir.exists());
String[] outFiles = outDir.list();
assertEquals("output dir files?", 4, outFiles.length);
- BufferedReader reader = new BufferedReader(new InputStreamReader(
- new FileInputStream("output/points/part-00000"), Charset
- .forName("UTF-8")));
DummyOutputCollector<Text, Text> collector = new DummyOutputCollector<Text, Text>();
- while (reader.ready()) {
- String line = reader.readLine();
+ for (String line : new FileLineIterable(new File("output/points/part-00000"))) {
String[] lineParts = line.split("\t");
assertEquals("line parts", 2, lineParts.length);
String cl = line.substring(0, line.indexOf(':'));
collector.collect(new Text(cl), new Text(lineParts[1]));
}
- reader.close();
assertEquals("num points[V0]", 4, collector.getValue("V0").size());
assertEquals("num points[V1]", 5, collector.getValue("V1").size());
}
Modified: lucene/mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/MahoutEvaluatorTest.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/MahoutEvaluatorTest.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/MahoutEvaluatorTest.java (original)
+++ lucene/mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/MahoutEvaluatorTest.java Wed Sep 9 15:41:31 2009
@@ -25,12 +25,12 @@
import org.apache.mahout.ga.watchmaker.utils.DummyCandidate;
import org.apache.mahout.ga.watchmaker.utils.DummyEvaluator;
import org.apache.mahout.utils.StringUtils;
+import org.apache.mahout.common.FileLineIterable;
-import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
+import java.util.Collection;
public class MahoutEvaluatorTest extends TestCase {
@@ -87,17 +87,10 @@
}
private static void loadPopulation(FileSystem fs, Path f,
- List<DummyCandidate> population) throws IOException {
+ Collection<DummyCandidate> population) throws IOException {
FSDataInputStream in = fs.open(f);
- BufferedReader reader = new BufferedReader(new InputStreamReader(in));
-
- try {
- String s;
- while ((s = reader.readLine()) != null) {
- population.add((DummyCandidate) StringUtils.fromString(s));
- }
- } finally {
- reader.close();
+ for (String line : new FileLineIterable(in)) {
+ population.add((DummyCandidate) StringUtils.fromString(line));
}
}
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/bookcrossing/BookCrossingDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/bookcrossing/BookCrossingDataModel.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/bookcrossing/BookCrossingDataModel.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/bookcrossing/BookCrossingDataModel.java Wed Sep 9 15:41:31 2009
@@ -18,8 +18,8 @@
package org.apache.mahout.cf.taste.example.bookcrossing;
import org.apache.mahout.cf.taste.example.grouplens.GroupLensDataModel;
-import org.apache.mahout.cf.taste.impl.common.FileLineIterable;
-import org.apache.mahout.cf.taste.impl.common.IOUtils;
+import org.apache.mahout.common.FileLineIterable;
+import org.apache.mahout.common.IOUtils;
import org.apache.mahout.cf.taste.impl.model.file.FileDataModel;
import java.io.File;
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/grouplens/GroupLensDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/grouplens/GroupLensDataModel.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/grouplens/GroupLensDataModel.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/grouplens/GroupLensDataModel.java Wed Sep 9 15:41:31 2009
@@ -17,8 +17,8 @@
package org.apache.mahout.cf.taste.example.grouplens;
-import org.apache.mahout.cf.taste.impl.common.FileLineIterable;
-import org.apache.mahout.cf.taste.impl.common.IOUtils;
+import org.apache.mahout.common.FileLineIterable;
+import org.apache.mahout.common.IOUtils;
import org.apache.mahout.cf.taste.impl.model.file.FileDataModel;
import java.io.File;
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/jester/JesterDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/jester/JesterDataModel.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/jester/JesterDataModel.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/jester/JesterDataModel.java Wed Sep 9 15:41:31 2009
@@ -22,7 +22,7 @@
import org.apache.mahout.cf.taste.impl.model.GenericDataModel;
import org.apache.mahout.cf.taste.impl.model.file.FileDataModel;
import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
-import org.apache.mahout.cf.taste.impl.common.FileLineIterator;
+import org.apache.mahout.common.FileLineIterator;
import org.apache.mahout.cf.taste.model.Preference;
import org.apache.mahout.cf.taste.model.DataModel;
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/NetflixDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/NetflixDataModel.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/NetflixDataModel.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/NetflixDataModel.java Wed Sep 9 15:41:31 2009
@@ -19,7 +19,7 @@
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.impl.common.FileLineIterable;
+import org.apache.mahout.common.FileLineIterable;
import org.apache.mahout.cf.taste.impl.common.FastIDSet;
import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
@@ -76,7 +76,7 @@
delegate = new GenericDataModel(users);
}
- private FastByIDMap<PreferenceArray> readUsers(File dataDirectory) {
+ private FastByIDMap<PreferenceArray> readUsers(File dataDirectory) throws IOException {
FastByIDMap<Collection<Preference>> userIDPrefMap = new FastByIDMap<Collection<Preference>>();
int counter = 0;
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/NetflixFileDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/NetflixFileDataModel.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/NetflixFileDataModel.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/NetflixFileDataModel.java Wed Sep 9 15:41:31 2009
@@ -18,9 +18,11 @@
package org.apache.mahout.cf.taste.example.netflix;
import org.apache.mahout.cf.taste.common.Refreshable;
-import org.apache.mahout.cf.taste.impl.common.FileLineIterable;
+import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.common.FileLineIterable;
import org.apache.mahout.cf.taste.impl.common.FastIDSet;
import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
+import org.apache.mahout.cf.taste.impl.common.AbstractLongPrimitiveIterator;
import org.apache.mahout.cf.taste.impl.model.GenericItemPreferenceArray;
import org.apache.mahout.cf.taste.impl.model.GenericPreference;
import org.apache.mahout.cf.taste.model.DataModel;
@@ -75,7 +77,7 @@
}
@Override
- public PreferenceArray getPreferencesForItem(long itemID) {
+ public PreferenceArray getPreferencesForItem(long itemID) throws TasteException {
StringBuilder itemIDPadded = new StringBuilder(5);
itemIDPadded.append(itemID);
while (itemIDPadded.length() < 5) {
@@ -83,12 +85,16 @@
}
List<Preference> prefs = new ArrayList<Preference>();
File movieFile = new File(new File(dataDirectory, "training_set"), "mv_00" + itemIDPadded + ".txt");
- for (String line : new FileLineIterable(movieFile, true)) {
- int firstComma = line.indexOf((int) ',');
- Integer userID = Integer.valueOf(line.substring(0, firstComma));
- int secondComma = line.indexOf((int) ',', firstComma + 1);
- float rating = Float.parseFloat(line.substring(firstComma + 1, secondComma));
- prefs.add(new GenericPreference(userID, itemID, rating));
+ try {
+ for (String line : new FileLineIterable(movieFile, true)) {
+ int firstComma = line.indexOf((int) ',');
+ Integer userID = Integer.valueOf(line.substring(0, firstComma));
+ int secondComma = line.indexOf((int) ',', firstComma + 1);
+ float rating = Float.parseFloat(line.substring(firstComma + 1, secondComma));
+ prefs.add(new GenericPreference(userID, itemID, rating));
+ }
+ } catch (IOException ioe) {
+ throw new TasteException(ioe);
}
return new GenericItemPreferenceArray(prefs);
}
@@ -134,7 +140,7 @@
return "NetflixFileDataModel";
}
- private static final class MovieIDIterator implements LongPrimitiveIterator {
+ private static final class MovieIDIterator extends AbstractLongPrimitiveIterator {
private int next = 1;
static final int COUNT = 17770;
@@ -161,13 +167,13 @@
}
@Override
- public Long next() {
- return nextLong();
+ public void remove() {
+ throw new UnsupportedOperationException();
}
@Override
- public void remove() {
- throw new UnsupportedOperationException();
+ public void skip(int n) {
+ next += n;
}
}
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/TransposeToByUser.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/TransposeToByUser.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/TransposeToByUser.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/netflix/TransposeToByUser.java Wed Sep 9 15:41:31 2009
@@ -18,8 +18,8 @@
package org.apache.mahout.cf.taste.example.netflix;
import org.apache.mahout.cf.taste.impl.common.FastMap;
-import org.apache.mahout.cf.taste.impl.common.FileLineIterable;
-import org.apache.mahout.cf.taste.impl.common.IOUtils;
+import org.apache.mahout.common.FileLineIterable;
+import org.apache.mahout.common.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java Wed Sep 9 15:41:31 2009
@@ -17,8 +17,6 @@
package org.apache.mahout.classifier.bayes;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.mahout.classifier.ClassifierResult;
import org.apache.mahout.classifier.ResultAnalyzer;
import org.apache.mahout.classifier.bayes.algorithm.BayesAlgorithm;
@@ -33,6 +31,7 @@
import org.apache.mahout.classifier.bayes.model.ClassifierContext;
import org.apache.mahout.utils.TimingStatistics;
import org.apache.mahout.utils.nlp.NGrams;
+import org.apache.mahout.common.FileLineIterable;
import org.apache.commons.cli2.Option;
import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
@@ -43,8 +42,13 @@
import org.apache.commons.cli2.builder.GroupBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
-import java.util.*;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.nio.charset.Charset;
public class TestClassifier {
@@ -139,8 +143,7 @@
String testDirPath = (String) cmdLine.getValue(dirOpt);
- String classificationMethod = "sequential";
- classificationMethod = (String) cmdLine.getValue(methodOpt);
+ String classificationMethod = (String) cmdLine.getValue(methodOpt);
params.set("verbose", Boolean.toString(verbose));
params.set("basePath", modelBasePath);
@@ -209,45 +212,37 @@
log.info("Testing: " + file);
String correctLabel = file.getName().split(".txt")[0];
final TimingStatistics operationStats = new TimingStatistics();
- BufferedReader fileReader = new BufferedReader(new InputStreamReader(
-
- new FileInputStream(file.getPath()), params.get("encoding")));
- try {
- String line;
- long lineNum = 0;
- while ((line = fileReader.readLine()) != null) {
-
- Map<String, List<String>> document = new NGrams(line, Integer.valueOf(params.get("gramSize")).intValue())
- .generateNGrams();
- for (Map.Entry<String, List<String>> stringListEntry : document.entrySet()) {
- List<String> strings = stringListEntry.getValue();
- TimingStatistics.Call call = operationStats.newCall();
- TimingStatistics.Call outercall = totalStatistics.newCall();
- ClassifierResult classifiedLabel = classifier.classifyDocument(strings
- .toArray(new String[strings.size()]), params.get("defaultCat"));
- call.end();
- outercall.end();
- boolean correct = resultAnalyzer.addInstance(correctLabel, classifiedLabel);
- if (verbose == true) {
- // We have one document per line
- log.info("Line Number: " + lineNum + " Line(30): "
- + (line.length() > 30 ? line.substring(0, 30) : line) + " Expected Label: " + correctLabel
- + " Classified Label: " + classifiedLabel.getLabel() + " Correct: " + correct);
- }
- // log.info("{} {}", correctLabel, classifiedLabel);
+ long lineNum = 0;
+ for (String line : new FileLineIterable(new File(file.getPath()), Charset.forName(params.get("encoding")), false)) {
+ Map<String, List<String>> document = new NGrams(line, Integer.parseInt(params.get("gramSize")))
+ .generateNGrams();
+ for (Map.Entry<String, List<String>> stringListEntry : document.entrySet()) {
+ List<String> strings = stringListEntry.getValue();
+ TimingStatistics.Call call = operationStats.newCall();
+ TimingStatistics.Call outercall = totalStatistics.newCall();
+ ClassifierResult classifiedLabel = classifier.classifyDocument(strings
+ .toArray(new String[strings.size()]), params.get("defaultCat"));
+ call.end();
+ outercall.end();
+ boolean correct = resultAnalyzer.addInstance(correctLabel, classifiedLabel);
+ if (verbose) {
+ // We have one document per line
+ log.info("Line Number: " + lineNum + " Line(30): "
+ + (line.length() > 30 ? line.substring(0, 30) : line) + " Expected Label: " + correctLabel
+ + " Classified Label: " + classifiedLabel.getLabel() + " Correct: " + correct);
}
- lineNum++;
+ // log.info("{} {}", correctLabel, classifiedLabel);
+
}
- log.info("{}\t{}\t{}/{}", new Object[] { correctLabel,
- resultAnalyzer.getConfusionMatrix().getAccuracy(correctLabel),
- resultAnalyzer.getConfusionMatrix().getCorrect(correctLabel),
- resultAnalyzer.getConfusionMatrix().getTotal(correctLabel) });
- log.info("{}", operationStats.toString());
- } finally {
- fileReader.close();
+ lineNum++;
}
+ log.info("{}\t{}\t{}/{}", new Object[] { correctLabel,
+ resultAnalyzer.getConfusionMatrix().getAccuracy(correctLabel),
+ resultAnalyzer.getConfusionMatrix().getCorrect(correctLabel),
+ resultAnalyzer.getConfusionMatrix().getTotal(correctLabel) });
+ log.info("{}", operationStats.toString());
}
}
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreatorDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreatorDriver.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreatorDriver.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreatorDriver.java Wed Sep 9 15:41:31 2009
@@ -37,13 +37,12 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.mahout.analysis.WikipediaAnalyzer;
import org.apache.mahout.utils.CommandLineUtil;
+import org.apache.mahout.common.FileLineIterable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
+import java.io.File;
import java.util.HashSet;
import java.util.Set;
@@ -61,7 +60,7 @@
* <ol>
* <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
* <li>The output {@link org.apache.hadoop.fs.Path} where to write the
- * {@link org.apache.mahout.classifier.bayes.BayesModel} as a {@link org.apache.hadoop.io.SequenceFile}</li>
+ * classifier as a {@link org.apache.hadoop.io.SequenceFile}</li>
* </ol>
*
* @param args The args
@@ -168,13 +167,9 @@
}
Set<String> categories = new HashSet<String>();
- BufferedReader reader = new BufferedReader(new InputStreamReader(
- new FileInputStream(catFile), "UTF-8"));
- String line;
- while ((line = reader.readLine()) != null) {
+ for (String line : new FileLineIterable(new File(catFile))) {
categories.add(line.trim().toLowerCase());
}
- reader.close();
DefaultStringifier<Set<String>> setStringifier = new DefaultStringifier<Set<String>>(conf, GenericsUtil.getClass(categories));
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java Wed Sep 9 15:41:31 2009
@@ -25,13 +25,11 @@
import org.apache.commons.cli2.builder.DefaultOptionBuilder;
import org.apache.commons.cli2.builder.ArgumentBuilder;
import org.apache.commons.cli2.builder.GroupBuilder;
+import org.apache.mahout.common.FileLineIterator;
-import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
-import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.IOException;
import java.text.DecimalFormat;
@@ -76,9 +74,6 @@
numChunks = Integer.parseInt((String) cmdLine.getValue(numChunksOpt));
}
- BufferedReader dumpReader = new BufferedReader(new InputStreamReader(
- new FileInputStream(dumpFilePath), "UTF-8"));
-
File dir = new File(outputDirPath);
dir.mkdirs();
String header =
@@ -115,14 +110,15 @@
content.append(header);
int filenumber = 0;
NumberFormat decimalFormatter = new DecimalFormat("0000");
- String thisLine;
- while ((thisLine = dumpReader.readLine()) != null)
- {
+ FileLineIterator it = new FileLineIterator(new File(dumpFilePath));
+ while (it.hasNext()) {
+ String thisLine = it.next();
if(thisLine.trim().startsWith("<page>")){
boolean end = false;
while(thisLine.trim().startsWith("</page>") == false){
content.append(thisLine).append('\n');
- if ((thisLine = dumpReader.readLine()) == null){
+ if (it.hasNext()){
+ thisLine = it.next();
end=true;
break;
}
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/dirichlet/DisplayASNOutputState.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/dirichlet/DisplayASNOutputState.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/dirichlet/DisplayASNOutputState.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/dirichlet/DisplayASNOutputState.java Wed Sep 9 15:41:31 2009
@@ -20,9 +20,7 @@
import java.awt.BasicStroke;
import java.awt.Graphics;
import java.awt.Graphics2D;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -36,6 +34,7 @@
import org.apache.mahout.matrix.DenseVector;
import org.apache.mahout.matrix.Vector;
import org.apache.mahout.common.RandomUtils;
+import org.apache.mahout.common.FileLineIterable;
class DisplayASNOutputState extends DisplayDirichlet {
DisplayASNOutputState() {
@@ -74,16 +73,11 @@
* if there is an error
*/
public static List<Vector> readFile(String fileName) throws IOException {
- BufferedReader r = new BufferedReader(new FileReader(fileName));
- try {
- List<Vector> results = new ArrayList<Vector>();
- String line;
- while ((line = r.readLine()) != null)
- results.add(AbstractVector.decodeVector(line));
- return results;
- } finally {
- r.close();
+ List<Vector> results = new ArrayList<Vector>();
+ for (String line : new FileLineIterable(new File(fileName))) {
+ results.add(AbstractVector.decodeVector(line));
}
+ return results;
}
private static void getSamples() throws IOException {
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/dirichlet/DisplayOutputState.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/dirichlet/DisplayOutputState.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/dirichlet/DisplayOutputState.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/dirichlet/DisplayOutputState.java Wed Sep 9 15:41:31 2009
@@ -20,9 +20,7 @@
import java.awt.BasicStroke;
import java.awt.Graphics;
import java.awt.Graphics2D;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -36,6 +34,7 @@
import org.apache.mahout.matrix.DenseVector;
import org.apache.mahout.matrix.Vector;
import org.apache.mahout.common.RandomUtils;
+import org.apache.mahout.common.FileLineIterable;
class DisplayOutputState extends DisplayDirichlet {
DisplayOutputState() {
@@ -73,16 +72,11 @@
* if there is an error
*/
public static List<Vector> readFile(String fileName) throws IOException {
- BufferedReader r = new BufferedReader(new FileReader(fileName));
- try {
- List<Vector> results = new ArrayList<Vector>();
- String line;
- while ((line = r.readLine()) != null)
- results.add(AbstractVector.decodeVector(line));
- return results;
- } finally {
- r.close();
+ List<Vector> results = new ArrayList<Vector>();
+ for (String line : new FileLineIterable(new File(fileName))) {
+ results.add(AbstractVector.decodeVector(line));
}
+ return results;
}
private static void getSamples() throws IOException {
Modified: lucene/mahout/trunk/examples/src/test/java/org/apache/mahout/ga/watchmaker/cd/FileInfosDatasetTest.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/test/java/org/apache/mahout/ga/watchmaker/cd/FileInfosDatasetTest.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/test/java/org/apache/mahout/ga/watchmaker/cd/FileInfosDatasetTest.java (original)
+++ lucene/mahout/trunk/examples/src/test/java/org/apache/mahout/ga/watchmaker/cd/FileInfosDatasetTest.java Wed Sep 9 15:41:31 2009
@@ -18,15 +18,14 @@
package org.apache.mahout.ga.watchmaker.cd;
import junit.framework.TestCase;
+import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.mahout.common.FileLineIterable;
-import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.FileInputStream;
-import java.nio.charset.Charset;
+import java.io.File;
public class FileInfosDatasetTest extends TestCase {
@@ -37,12 +36,8 @@
DataSet dataset = FileInfoParser.parseFile(fs, inpath);
DataSet.initialize(dataset);
- String filename = "target/test-classes/wdbc/wdbc.data";
- BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(filename), Charset.forName("UTF-8")));
-
- String line;
DataLine dl = new DataLine();
- while ((line = in.readLine()) != null) {
+ for (String line : new FileLineIterable(new File("target/test-classes/wdbc/wdbc.data"))) {
dl.set(line);
for (int index = 0; index < dataset.getNbAttributes(); index++) {
if (dataset.isNumerical(index)) {
@@ -53,12 +48,11 @@
}
}
}
- in.close();
}
- private void assertInRange(double value, double min, double max) {
- TestCase.assertTrue("value"+value+") < min", value >= min);
- TestCase.assertTrue("value("+value+") > max", value <= max);
+ private static void assertInRange(double value, double min, double max) {
+ Assert.assertTrue("value" + value + ") < min", value >= min);
+ Assert.assertTrue("value(" + value + ") > max", value <= max);
}
}
Modified: lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java (original)
+++ lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java Wed Sep 9 15:41:31 2009
@@ -36,12 +36,11 @@
import org.apache.mahout.clustering.ClusterBase;
import org.apache.mahout.matrix.Vector;
import org.apache.mahout.utils.CommandLineUtil;
+import org.apache.mahout.common.FileLineIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
@@ -219,9 +218,9 @@
}
private static ArrayList<String> getTermDict(String dictFile) throws IOException {
- BufferedReader reader = new BufferedReader(new FileReader(dictFile));
-
- int numEntries = Integer.parseInt(reader.readLine());
+ FileLineIterator it = new FileLineIterator(new File(dictFile));
+
+ int numEntries = Integer.parseInt(it.next());
System.out.println(numEntries);
ArrayList<String> result = new ArrayList<String>();
@@ -229,8 +228,8 @@
result.add("dummyentry");
}
- String line;
- while ((line = reader.readLine()) != null) {
+ while (it.hasNext()) {
+ String line = it.next();
if (line.startsWith("#")) {
continue;
}
Modified: lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java?rev=813003&r1=813002&r2=813003&view=diff
==============================================================================
--- lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java (original)
+++ lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java Wed Sep 9 15:41:31 2009
@@ -20,7 +20,7 @@
import org.apache.mahout.matrix.DenseVector;
import org.apache.mahout.matrix.SparseVector;
import org.apache.mahout.matrix.Vector;
-import org.apache.mahout.utils.vectors.VectorIterable;
+import org.apache.mahout.common.IOUtils;
import java.io.BufferedReader;
import java.io.File;
@@ -34,33 +34,30 @@
import java.text.SimpleDateFormat;
import java.text.DateFormat;
import java.util.Iterator;
+import java.util.NoSuchElementException;
import java.util.regex.Pattern;
-
/**
* Read in ARFF (http://www.cs.waikato.ac.nz/~ml/weka/arff.html) and create {@link Vector}s
* <p/>
* Attribute type handling:
* <ul>
* <li>Numeric -> As is</li>
- * <li>Nominal -> ordinal(value) i.e. @attribute lumber {'\'(-inf-0.5]\'','\'(0.5-inf)\''} will convert -inf-0.5 -> 0, and 0.5-inf -> 1</li>
+ * <li>Nominal -> ordinal(value) i.e. @attribute lumber {'\'(-inf-0.5]\'','\'(0.5-inf)\''}
+ * will convert -inf-0.5 -> 0, and 0.5-inf -> 1</li>
* <li>Dates -> Convert to time as a long</li>
* <li>Strings -> Create a map of String -> long</li>
* </ul>
- * <p/>
- * <p/>
- * <p/>
- * <p/>
* NOTE: This class does not set the label bindings on every vector. If you want the label
* bindings, call {@link MapBackedARFFModel#getLabelBindings()}, as they are the same for every vector.
*/
public class ARFFVectorIterable implements Iterable<Vector> {
- private final BufferedReader buff;
- private final ARFFModel model;
private static final Pattern COMMA_PATTERN = Pattern.compile(",");
private static final Pattern SPACE_PATTERN = Pattern.compile(" ");
+ private final BufferedReader buff;
+ private final ARFFModel model;
public ARFFVectorIterable(File file, ARFFModel model) throws IOException {
this(new FileReader(file), model);
@@ -85,7 +82,7 @@
int labelNumber = 0;
String line;
- boolean inData = false;
+ //boolean inData = false; // TODO not used?
while ((line = buff.readLine()) != null) {
line = line.trim();
String lower = line.toLowerCase();
@@ -138,7 +135,7 @@
model.addType(labelNumInt, type);
labelNumber++;
} else if (lower.startsWith(ARFFModel.DATA)) {
- inData = true;
+ //inData = true;
break;//skip it
}
}
@@ -146,21 +143,21 @@
}
- /*public ARFFVectorIterable(SequenceFile seqFile){
-
- }*/
-
@Override
public Iterator<Vector> iterator() {
return new ARFFIterator();
}
- class ARFFIterator implements Iterator<Vector> {
- String line = null;
+ private class ARFFIterator implements Iterator<Vector> {
- @Override
- public boolean hasNext() {
- boolean result = false;
+ private String line;
+
+ private ARFFIterator() {
+ goToNext();
+ }
+
+ private void goToNext() {
+ line = null;
try {
while ((line = buff.readLine()) != null) {
line = line.trim();
@@ -168,17 +165,24 @@
break;
}
}
- if (line != null) {
- result = true;
- }
} catch (IOException e) {
- result = false;
+ line = null;
+ }
+ if (line == null) {
+ IOUtils.quietClose(buff);
}
- return result;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return line != null;
}
@Override
public Vector next() {
+ if (line == null) {
+ throw new NoSuchElementException();
+ }
Vector result;
if (line.startsWith(ARFFModel.ARFF_SPARSE)) {
line = line.substring(1, line.length() - 1);
@@ -197,11 +201,10 @@
}
}
//result.setLabelBindings(labelBindings);
+ goToNext();
return result;
}
-
-
@Override
public void remove() {
throw new UnsupportedOperationException("remove not supported");