You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2011/04/12 20:55:26 UTC
svn commit: r1091538 [1/2] - in /mahout/trunk:
core/src/main/java/org/apache/mahout/cf/taste/hadoop/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/
core/src/main/java/org/apache/mahout/cf/taste/impl/model/
core/src/main/java/org/...
Author: srowen
Date: Tue Apr 12 18:55:25 2011
New Revision: 1091538
URL: http://svn.apache.org/viewvc?rev=1091538&view=rev
Log:
More use of Guava, more tweaks like copyright headers
Added:
mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/CountDescendingPairComparator.java
Modified:
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/EntityEntityWritable.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/TasteHadoopUtils.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/CountUsersKeyWritable.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/AbstractIDMigrator.java
mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java
mahout/trunk/core/src/main/java/org/apache/mahout/classifier/Classify.java
mahout/trunk/core/src/main/java/org/apache/mahout/classifier/OnlineLearner.java
mahout/trunk/core/src/main/java/org/apache/mahout/common/LongPair.java
mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/StableFixedSizeSamplingIterator.java
mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowth.java
mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/ParallelFPGrowthReducer.java
mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/convertors/string/TopKStringPatterns.java
mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/FPGrowth.java
mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/Pattern.java
mahout/trunk/core/src/main/java/org/apache/mahout/math/VarLongWritable.java
mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java
mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/Gram.java
mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJobTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/classifier/naivebayes/NaiveBayesTestBase.java
mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sequencelearning/hmm/HMMTestBase.java
mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/OnlineLogisticRegressionTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestDistributions.java
mahout/trunk/core/src/test/java/org/apache/mahout/df/data/Utils.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/RandomDocumentGenerator.java
mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/grouplens/GroupLensDataModel.java
mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/kddcup/track1/Track1RecommenderEvaluator.java
mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/kddcup/track1/svd/Track1SVDRunner.java
mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/SplitBayesInput.java
mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sequencelearning/hmm/PosTagger.java
mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/SimpleCsvExamples.java
mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/TrainNewsGroups.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplaySpectralKMeans.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/minhash/LastfmDataConverter.java
mahout/trunk/math/src/main/java/org/apache/mahout/common/DevURandomSeedGenerator.java
mahout/trunk/math/src/main/java/org/apache/mahout/common/RandomUtils.java
mahout/trunk/math/src/main/java/org/apache/mahout/common/RandomWrapper.java
mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java
mahout/trunk/math/src/test/java/org/apache/mahout/common/RandomUtilsTest.java
mahout/trunk/taste-web/src/main/java/org/apache/mahout/cf/taste/web/RecommenderWrapper.java
mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/lda/LDAPrintTopics.java
mahout/trunk/utils/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java
mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java
mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java
mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java
mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java
mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java
mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/EntityEntityWritable.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/EntityEntityWritable.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/EntityEntityWritable.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/EntityEntityWritable.java Tue Apr 12 18:55:25 2011
@@ -21,8 +21,8 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import com.google.common.primitives.Longs;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.math.Varint;
/** A {@link WritableComparable} encapsulating two items. */
@@ -78,7 +78,7 @@ public final class EntityEntityWritable
@Override
public int hashCode() {
- return RandomUtils.hashLong(aID) + 31 * RandomUtils.hashLong(bID);
+ return Longs.hashCode(aID) + 31 * Longs.hashCode(bID);
}
@Override
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/TasteHadoopUtils.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/TasteHadoopUtils.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/TasteHadoopUtils.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/TasteHadoopUtils.java Tue Apr 12 18:55:25 2011
@@ -18,6 +18,7 @@
package org.apache.mahout.cf.taste.hadoop;
import com.google.common.base.Charsets;
+import com.google.common.primitives.Longs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -57,7 +58,7 @@ public final class TasteHadoopUtils {
* Maps a long to an int
*/
public static int idToIndex(long id) {
- return 0x7FFFFFFF & ((int) id ^ (int) (id >>> 32));
+ return 0x7FFFFFFF & Longs.hashCode(id);
}
/**
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/CountUsersKeyWritable.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/CountUsersKeyWritable.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/CountUsersKeyWritable.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/CountUsersKeyWritable.java Tue Apr 12 18:55:25 2011
@@ -17,10 +17,10 @@
package org.apache.mahout.cf.taste.hadoop.similarity.item;
+import com.google.common.primitives.Longs;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.Partitioner;
-import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.math.VarLongWritable;
import org.apache.mahout.math.Varint;
@@ -71,7 +71,7 @@ public class CountUsersKeyWritable imple
@Override
public int hashCode() {
- return RandomUtils.hashLong(userID);
+ return Longs.hashCode(userID);
}
/**
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/AbstractIDMigrator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/AbstractIDMigrator.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/AbstractIDMigrator.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/AbstractIDMigrator.java Tue Apr 12 18:55:25 2011
@@ -17,18 +17,17 @@
package org.apache.mahout.cf.taste.impl.model;
-import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Collection;
+
+import com.google.common.base.Charsets;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.model.IDMigrator;
public abstract class AbstractIDMigrator implements IDMigrator {
-
- private static final Charset UTF8_CHARSET = Charset.forName("UTF8");
-
+
private final MessageDigest md5Digest;
protected AbstractIDMigrator() {
@@ -46,7 +45,7 @@ public abstract class AbstractIDMigrator
protected final long hash(String value) {
byte[] md5hash;
synchronized (md5Digest) {
- md5hash = md5Digest.digest(value.getBytes(UTF8_CHARSET));
+ md5hash = md5Digest.digest(value.getBytes(Charsets.UTF_8));
md5Digest.reset();
}
long hash = 0L;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/classifier/BayesFileFormatter.java Tue Apr 12 18:55:25 2011
@@ -19,11 +19,7 @@ package org.apache.mahout.classifier;
import java.io.File;
import java.io.FileFilter;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.nio.charset.Charset;
@@ -31,6 +27,7 @@ import java.util.ArrayList;
import java.util.List;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
import org.apache.commons.cli2.Option;
@@ -75,7 +72,7 @@ public final class BayesFileFormatter {
*/
public static void collapse(String label, Analyzer analyzer, File inputDir,
Charset charset, File outputFile) throws IOException {
- Writer writer = new OutputStreamWriter(new FileOutputStream(outputFile), charset);
+ Writer writer = Files.newWriter(outputFile, charset);
try {
inputDir.listFiles(new FileProcessor(label, analyzer, charset, writer));
// listFiles() is called here as a way to recursively visit files,
@@ -104,7 +101,7 @@ public final class BayesFileFormatter {
if (input.isDirectory()) {
input.listFiles(new FileProcessor(label, analyzer, charset, outDir));
} else {
- Writer writer = new OutputStreamWriter(new FileOutputStream(new File(outDir, input.getName())), charset);
+ Writer writer = Files.newWriter(new File(outDir, input.getName()), charset);
try {
writeFile(label, analyzer, input, charset, writer);
} finally {
@@ -162,7 +159,7 @@ public final class BayesFileFormatter {
Writer theWriter = null;
try {
if (writer == null) {
- theWriter = new OutputStreamWriter(new FileOutputStream(new File(outputDir, file.getName())), charset);
+ theWriter = Files.newWriter(new File(outputDir, file.getName()), charset);
} else {
theWriter = writer;
}
@@ -204,7 +201,7 @@ public final class BayesFileFormatter {
*/
private static void writeFile(String label, Analyzer analyzer, File inFile,
Charset charset, Writer writer) throws IOException {
- Reader reader = new InputStreamReader(new FileInputStream(inFile), charset);
+ Reader reader = Files.newReader(inFile, charset);
try {
TokenStream ts = analyzer.tokenStream(label, reader);
writer.write(label);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/classifier/Classify.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/Classify.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/classifier/Classify.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/classifier/Classify.java Tue Apr 12 18:55:25 2011
@@ -18,11 +18,10 @@
package org.apache.mahout.classifier;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.List;
+import com.google.common.io.Files;
import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
import org.apache.commons.cli2.Option;
@@ -156,8 +155,9 @@ public final class Classify {
}
log.info("Converting input document to proper format");
- String[] document = BayesFileFormatter.readerToDocument(analyzer, new InputStreamReader(
- new FileInputStream(docPath), Charset.forName(encoding)));
+
+ String[] document =
+ BayesFileFormatter.readerToDocument(analyzer,Files.newReader(docPath, Charset.forName(encoding)));
StringBuilder line = new StringBuilder();
for (String token : document) {
line.append(token).append(' ');
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/classifier/OnlineLearner.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/OnlineLearner.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/classifier/OnlineLearner.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/classifier/OnlineLearner.java Tue Apr 12 18:55:25 2011
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.mahout.classifier;
import org.apache.mahout.math.Vector;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/common/LongPair.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/LongPair.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/LongPair.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/LongPair.java Tue Apr 12 18:55:25 2011
@@ -19,6 +19,8 @@ package org.apache.mahout.common;
import java.io.Serializable;
+import com.google.common.primitives.Longs;
+
/** A simple (ordered) pair of longs. */
public final class LongPair implements Comparable<LongPair>, Serializable {
@@ -53,10 +55,10 @@ public final class LongPair implements C
@Override
public int hashCode() {
- int firstHash = RandomUtils.hashLong(first);
+ int firstHash = Longs.hashCode(first);
// Flip top and bottom 16 bits; this makes the hash function probably different
// for (a,b) versus (b,a)
- return (firstHash >>> 16 | firstHash << 16) ^ RandomUtils.hashLong(second);
+ return (firstHash >>> 16 | firstHash << 16) ^ Longs.hashCode(second);
}
@Override
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/StableFixedSizeSamplingIterator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/StableFixedSizeSamplingIterator.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/StableFixedSizeSamplingIterator.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/StableFixedSizeSamplingIterator.java Tue Apr 12 18:55:25 2011
@@ -19,7 +19,6 @@ package org.apache.mahout.common.iterato
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
@@ -55,19 +54,14 @@ public class StableFixedSizeSamplingIter
}
}
- Collections.sort(buf, new Comparator<Pair<Integer,T>>() {
- @Override
- public int compare(Pair<Integer,T> pair1, Pair<Integer,T> pair2) {
- return pair1.getFirst().compareTo(pair2.getFirst());
- }
- });
+ Collections.sort(buf);
delegate = Iterators.transform(buf.iterator(),
- new Function<Pair<Integer,T>,T>() {
- @Override
- public T apply(Pair<Integer,T> from) {
- return from.getSecond();
- }
- });
+ new Function<Pair<Integer,T>,T>() {
+ @Override
+ public T apply(Pair<Integer,T> from) {
+ return from.getSecond();
+ }
+ });
}
@Override
Added: mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/CountDescendingPairComparator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/CountDescendingPairComparator.java?rev=1091538&view=auto
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/CountDescendingPairComparator.java (added)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/CountDescendingPairComparator.java Tue Apr 12 18:55:25 2011
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.fpm.pfpgrowth;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+import org.apache.mahout.common.Pair;
+
+/**
+ * Defines an ordering on {@link Pair}s whose second element is a count. The ordering places those with
+ * high count first (that is, descending), and for those of equal count, orders by the first element in the
+ * pair, ascending. It is used in several places in the FPM code.
+ */
+public final class CountDescendingPairComparator<A extends Comparable<? super A>,B extends Comparable<? super B>>
+ implements Comparator<Pair<A,B>>, Serializable {
+
+ @Override
+ public int compare(Pair<A,B> a, Pair<A,B> b) {
+ int ret = b.getSecond().compareTo(a.getSecond());
+ if (ret != 0) {
+ return ret;
+ }
+ return a.getFirst().compareTo(b.getFirst());
+ }
+}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowth.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowth.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowth.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowth.java Tue Apr 12 18:55:25 2011
@@ -19,7 +19,6 @@ package org.apache.mahout.fpm.pfpgrowth;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -127,19 +126,8 @@ public final class PFPGrowth {
int minSupport = Integer.valueOf(params.get(MIN_SUPPORT, "3"));
Configuration conf = new Configuration();
- PriorityQueue<Pair<String,Long>> queue = new PriorityQueue<Pair<String,Long>>(11,
- new Comparator<Pair<String,Long>>() {
-
- @Override
- public int compare(Pair<String,Long> o1, Pair<String,Long> o2) {
- int ret = o2.getSecond().compareTo(o1.getSecond());
- if (ret != 0) {
- return ret;
- }
- return o1.getFirst().compareTo(o2.getFirst());
- }
-
- });
+ PriorityQueue<Pair<String,Long>> queue =
+ new PriorityQueue<Pair<String,Long>>(11, new CountDescendingPairComparator<String,Long>());
Path parallelCountingPath = new Path(params.get(OUTPUT), PARALLEL_COUNTING);
Path filesPattern = new Path(parallelCountingPath, FILE_PATTERN);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/ParallelFPGrowthReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/ParallelFPGrowthReducer.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/ParallelFPGrowthReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/ParallelFPGrowthReducer.java Tue Apr 12 18:55:25 2011
@@ -20,7 +20,6 @@ package org.apache.mahout.fpm.pfpgrowth;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -73,18 +72,7 @@ public class ParallelFPGrowthReducer ext
}
- Collections.sort(localFList, new Comparator<Pair<Integer,Long>>() {
-
- @Override
- public int compare(Pair<Integer,Long> o1, Pair<Integer,Long> o2) {
- int ret = o2.getSecond().compareTo(o1.getSecond());
- if (ret != 0) {
- return ret;
- }
- return o1.getFirst().compareTo(o2.getFirst());
- }
-
- });
+ Collections.sort(localFList, new CountDescendingPairComparator<Integer,Long>());
FPGrowth<Integer> fpGrowth = new FPGrowth<Integer>();
fpGrowth.generateTopKFrequentPatterns(
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/convertors/string/TopKStringPatterns.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/convertors/string/TopKStringPatterns.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/convertors/string/TopKStringPatterns.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/convertors/string/TopKStringPatterns.java Tue Apr 12 18:55:25 2011
@@ -27,7 +27,6 @@ import java.util.List;
import org.apache.hadoop.io.Writable;
import org.apache.mahout.common.Pair;
-import com.google.common.base.Charsets;
/**
* A class which collects Top K string patterns
@@ -112,10 +111,7 @@ public final class TopKStringPatterns im
int itemsetLength = in.readInt();
long support = in.readLong();
for (int j = 0; j < itemsetLength; j++) {
- int itemLength = in.readInt();
- byte[] data = new byte[itemLength];
- in.readFully(data);
- items.add(new String(data, Charsets.UTF_8));
+ items.add(in.readUTF());
}
frequentPatterns.add(new Pair<List<String>,Long>(items, support));
}
@@ -128,9 +124,7 @@ public final class TopKStringPatterns im
out.writeInt(pattern.getFirst().size());
out.writeLong(pattern.getSecond());
for (String item : pattern.getFirst()) {
- byte[] data = item.getBytes(Charsets.UTF_8);
- out.writeInt(data.length);
- out.write(data);
+ out.writeUTF(item);
}
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/FPGrowth.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/FPGrowth.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/FPGrowth.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/FPGrowth.java Tue Apr 12 18:55:25 2011
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -37,6 +36,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.mahout.common.Pair;
import org.apache.mahout.common.iterator.sequencefile.SequenceFileIterable;
+import org.apache.mahout.fpm.pfpgrowth.CountDescendingPairComparator;
import org.apache.mahout.fpm.pfpgrowth.convertors.StatusUpdater;
import org.apache.mahout.fpm.pfpgrowth.convertors.TopKPatternsOutputConverter;
import org.apache.mahout.fpm.pfpgrowth.convertors.TransactionIterator;
@@ -78,13 +78,11 @@ public class FPGrowth<A extends Comparab
public final List<Pair<A,Long>> generateFList(Iterator<Pair<List<A>,Long>> transactions, int minSupport) {
Map<A,MutableLong> attributeSupport = new HashMap<A,MutableLong>();
- // int count = 0;
while (transactions.hasNext()) {
Pair<List<A>,Long> transaction = transactions.next();
for (A attribute : transaction.getFirst()) {
if (attributeSupport.containsKey(attribute)) {
attributeSupport.get(attribute).add(transaction.getSecond().longValue());
- // count++;
} else {
attributeSupport.put(attribute, new MutableLong(transaction.getSecond()));
}
@@ -98,18 +96,7 @@ public class FPGrowth<A extends Comparab
}
}
- Collections.sort(fList, new Comparator<Pair<A,Long>>() {
-
- @Override
- public int compare(Pair<A,Long> o1, Pair<A,Long> o2) {
- int ret = o2.getSecond().compareTo(o1.getSecond());
- if (ret != 0) {
- return ret;
- }
- return o1.getFirst().compareTo(o2.getFirst());
- }
-
- });
+ Collections.sort(fList, new CountDescendingPairComparator<A,Long>());
return fList;
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/Pattern.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/Pattern.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/Pattern.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/Pattern.java Tue Apr 12 18:55:25 2011
@@ -17,7 +17,7 @@
package org.apache.mahout.fpm.pfpgrowth.fpgrowth;
-import org.apache.mahout.common.RandomUtils;
+import com.google.common.primitives.Longs;
import java.util.Arrays;
@@ -139,7 +139,7 @@ public class Pattern implements Comparab
return hashCode;
}
int result = Arrays.hashCode(pattern);
- result = 31 * result + RandomUtils.hashLong(support);
+ result = 31 * result + Longs.hashCode(support);
result = 31 * result + length;
hashCode = result;
return result;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/VarLongWritable.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/VarLongWritable.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/VarLongWritable.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/VarLongWritable.java Tue Apr 12 18:55:25 2011
@@ -21,6 +21,7 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import com.google.common.primitives.Longs;
import org.apache.hadoop.io.WritableComparable;
public class VarLongWritable implements WritableComparable<VarLongWritable>, Cloneable {
@@ -49,7 +50,7 @@ public class VarLongWritable implements
@Override
public int hashCode() {
- return (int) ((value >>> 32) ^ value);
+ return Longs.hashCode(value);
}
@Override
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java Tue Apr 12 18:55:25 2011
@@ -225,9 +225,15 @@ public class EigenVerificationJob extend
Collections.sort(prunedEigenMeta, new Comparator<Map.Entry<MatrixSlice, EigenStatus>>() {
@Override
- public int compare(Map.Entry<MatrixSlice, EigenStatus> e1,
- Map.Entry<MatrixSlice, EigenStatus> e2) {
- return e1.getKey().index() - e2.getKey().index();
+ public int compare(Map.Entry<MatrixSlice,EigenStatus> e1, Map.Entry<MatrixSlice,EigenStatus> e2) {
+ int index1 = e1.getKey().index();
+ int index2 = e2.getKey().index();
+ if (index1 < index2) {
+ return -1;
+ } else if (index1 > index2) {
+ return 1;
+ }
+ return 0;
}
});
return prunedEigenMeta;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/Gram.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/Gram.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/Gram.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/Gram.java Tue Apr 12 18:55:25 2011
@@ -63,14 +63,11 @@ public class Gram extends BinaryComparab
/**
* Copy constructor
- *
- * @param other
*/
public Gram(Gram other) {
frequency = other.frequency;
length = other.length;
- bytes = new byte[length];
- System.arraycopy(other.bytes, 0, bytes, 0, length);
+ bytes = other.bytes.clone();
}
/**
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJobTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJobTest.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJobTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJobTest.java Tue Apr 12 18:55:25 2011
@@ -19,14 +19,12 @@ package org.apache.mahout.cf.taste.hadoo
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FilenameFilter;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.List;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
@@ -213,8 +211,7 @@ public final class ItemSimilarityJobTest
return name.startsWith("part-");
}
})[0];
- BufferedReader reader = new BufferedReader(
- new InputStreamReader(new FileInputStream(outPart), Charsets.UTF_8));
+ BufferedReader reader = Files.newReader(outPart, Charsets.UTF_8);
String line;
int currentLine = 1;
@@ -310,8 +307,7 @@ public final class ItemSimilarityJobTest
return name.startsWith("part-");
}
})[0];
- BufferedReader reader = new BufferedReader(
- new InputStreamReader(new FileInputStream(outPart), Charsets.UTF_8));
+ BufferedReader reader = Files.newReader(outPart, Charsets.UTF_8);
String line;
int currentLine = 1;
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierSelfTest.java Tue Apr 12 18:55:25 2011
@@ -19,12 +19,10 @@ package org.apache.mahout.classifier.bay
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileOutputStream;
-import java.io.OutputStreamWriter;
-import java.nio.charset.Charset;
import java.util.List;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -52,8 +50,7 @@ public final class BayesClassifierSelfTe
super.setUp();
File tempInputFile = getTestTempFile("bayesinput");
- BufferedWriter writer = new BufferedWriter(
- new OutputStreamWriter(new FileOutputStream(tempInputFile), Charsets.UTF_8));
+ BufferedWriter writer = Files.newWriter(tempInputFile, Charsets.UTF_8);
for (String[] entry : ClassifierData.DATA) {
writer.write(entry[0] + '\t' + entry[1] + '\n');
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesFileFormatterTest.java Tue Apr 12 18:55:25 2011
@@ -18,12 +18,11 @@
package org.apache.mahout.classifier.bayes;
import java.io.File;
-import java.io.FileOutputStream;
-import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.Iterator;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.mahout.classifier.BayesFileFormatter;
@@ -48,7 +47,7 @@ public final class BayesFileFormatterTes
out = getTestTempDir("bayes/out");
for (String word : WORDS) {
File file = new File(input, word);
- Writer writer = new OutputStreamWriter(new FileOutputStream(file), Charsets.UTF_8);
+ Writer writer = Files.newWriter(file, Charsets.UTF_8);
writer.write(word);
writer.close();
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/naivebayes/NaiveBayesTestBase.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/naivebayes/NaiveBayesTestBase.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/naivebayes/NaiveBayesTestBase.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/naivebayes/NaiveBayesTestBase.java Tue Apr 12 18:55:25 2011
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.mahout.classifier.naivebayes;
import java.util.Iterator;
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sequencelearning/hmm/HMMTestBase.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sequencelearning/hmm/HMMTestBase.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sequencelearning/hmm/HMMTestBase.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sequencelearning/hmm/HMMTestBase.java Tue Apr 12 18:55:25 2011
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.mahout.classifier.sequencelearning.hmm;
import org.apache.mahout.common.MahoutTestCase;
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/OnlineLogisticRegressionTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/OnlineLogisticRegressionTest.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/OnlineLogisticRegressionTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/OnlineLogisticRegressionTest.java Tue Apr 12 18:55:25 2011
@@ -18,6 +18,7 @@
package org.apache.mahout.classifier.sgd;
import com.google.common.base.CharMatcher;
+import com.google.common.base.Charsets;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -244,7 +245,7 @@ public final class OnlineLogisticRegress
private static Matrix readCsv(String resourceName) throws IOException {
Splitter onCommas = Splitter.on(",").trimResults(CharMatcher.anyOf(" \""));
- Readable isr = new InputStreamReader(Resources.getResource(resourceName).openStream());
+ Readable isr = new InputStreamReader(Resources.getResource(resourceName).openStream(), Charsets.UTF_8);
List<String> data = CharStreams.readLines(isr);
String first = data.get(0);
data = data.subList(1, data.size());
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestDistributions.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestDistributions.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestDistributions.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestDistributions.java Tue Apr 12 18:55:25 2011
@@ -53,7 +53,7 @@ public final class TestDistributions ext
for (int i = -30; i < 30; i++) {
double d = (i * 0.1);
double dnorm = UncommonDistributions.dNorm(d, 0, 1);
- byte[] bar = new byte[(int) (dnorm * 100)];
+ char[] bar = new char[(int) (dnorm * 100)];
for (int j = 0; j < bar.length; j++) {
bar[j] = '*';
}
@@ -67,7 +67,7 @@ public final class TestDistributions ext
for (int i = -30; i < 30; i++) {
double d = (i * 0.1);
double dnorm = UncommonDistributions.dNorm(d, 0, 2);
- byte[] bar = new byte[(int) (dnorm * 100)];
+ char[] bar = new char[(int) (dnorm * 100)];
for (int j = 0; j < bar.length; j++) {
bar[j] = '*';
}
@@ -81,7 +81,7 @@ public final class TestDistributions ext
for (int i = -10; i < 10; i++) {
double d = (i * 0.1);
double dnorm = UncommonDistributions.dNorm(d, 0, 0.2);
- byte[] bar = new byte[(int) (dnorm * 100)];
+ char[] bar = new char[(int) (dnorm * 100)];
for (int j = 0; j < bar.length; j++) {
bar[j] = '*';
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/df/data/Utils.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/df/data/Utils.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/df/data/Utils.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/df/data/Utils.java Tue Apr 12 18:55:25 2011
@@ -18,14 +18,13 @@
package org.apache.mahout.df.data;
import java.io.BufferedWriter;
-import java.io.FileOutputStream;
+import java.io.File;
import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Random;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -247,8 +246,7 @@ public final class Utils {
}
private static void writeDataToFile(String[] sData, Path path) throws IOException {
- BufferedWriter output = new BufferedWriter(new OutputStreamWriter(
- new FileOutputStream(path.toString()), Charsets.UTF_8));
+ BufferedWriter output = Files.newWriter(new File(path.toString()), Charsets.UTF_8);
try {
for (String line : sData) {
output.write(line);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java Tue Apr 12 18:55:25 2011
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.mahout.vectorizer;
import org.apache.hadoop.conf.Configuration;
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/RandomDocumentGenerator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/RandomDocumentGenerator.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/RandomDocumentGenerator.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/RandomDocumentGenerator.java Tue Apr 12 18:55:25 2011
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.mahout.vectorizer;
import java.util.Random;
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/grouplens/GroupLensDataModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/grouplens/GroupLensDataModel.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/grouplens/GroupLensDataModel.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/grouplens/GroupLensDataModel.java Tue Apr 12 18:55:25 2011
@@ -21,12 +21,15 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
+import java.net.URL;
import java.util.regex.Pattern;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.google.common.io.InputSupplier;
+import com.google.common.io.Resources;
import org.apache.mahout.cf.taste.impl.model.file.FileDataModel;
import org.apache.mahout.common.iterator.FileLineIterable;
import org.apache.mahout.common.IOUtils;
@@ -75,34 +78,22 @@ public final class GroupLensDataModel ex
}
return resultFile;
}
-
+
public static File readResourceToTempFile(String resourceName) throws IOException {
- InputStream is = GroupLensRecommender.class.getResourceAsStream(resourceName);
- if (is == null) {
- // No resource found, try just using the file
- return new File("src/main/java" + resourceName);
- }
+ InputSupplier<? extends InputStream> inSupplier;
try {
- File tempFile = File.createTempFile("taste", null);
- tempFile.deleteOnExit();
- OutputStream os = new FileOutputStream(tempFile);
- try {
- int bytesRead;
- byte[] buffer = new byte[32768];
- while ((bytesRead = is.read(buffer)) > 0) {
- os.write(buffer, 0, bytesRead);
- }
- os.flush();
- return tempFile;
- } finally {
- IOUtils.quietClose(os);
- }
- } finally {
- IOUtils.quietClose(is);
+ URL resourceURL = Resources.getResource(GroupLensRecommender.class, resourceName);
+ inSupplier = Resources.newInputStreamSupplier(resourceURL);
+ } catch (IllegalArgumentException iae) {
+ File resourceFile = new File("src/main/java" + resourceName);
+ inSupplier = Files.newInputStreamSupplier(resourceFile);
}
+ File tempFile = File.createTempFile("taste", null);
+ tempFile.deleteOnExit();
+ Files.copy(inSupplier, tempFile);
+ return tempFile;
}
-
-
+
@Override
public String toString() {
return "GroupLensDataModel";
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/kddcup/track1/Track1RecommenderEvaluator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/kddcup/track1/Track1RecommenderEvaluator.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/kddcup/track1/Track1RecommenderEvaluator.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/kddcup/track1/Track1RecommenderEvaluator.java Tue Apr 12 18:55:25 2011
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.mahout.cf.taste.example.kddcup.track1;
import java.io.File;
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/kddcup/track1/svd/Track1SVDRunner.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/kddcup/track1/svd/Track1SVDRunner.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/kddcup/track1/svd/Track1SVDRunner.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/cf/taste/example/kddcup/track1/svd/Track1SVDRunner.java Tue Apr 12 18:55:25 2011
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.mahout.cf.taste.example.kddcup.track1.svd;
import org.apache.mahout.cf.taste.common.NoSuchItemException;
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/SplitBayesInput.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/SplitBayesInput.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/SplitBayesInput.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/SplitBayesInput.java Tue Apr 12 18:55:25 2011
@@ -342,7 +342,7 @@ public class SplitBayesInput {
new Object[] {inputFile, testSplitSize, lineCount - testSplitSize});
}
}
-
+
BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(inputFile), charset));
Writer trainingWriter = new OutputStreamWriter(fs.create(trainingOutputFile), charset);
Writer testWriter = new OutputStreamWriter(fs.create(testOutputFile), charset);
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sequencelearning/hmm/PosTagger.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sequencelearning/hmm/PosTagger.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sequencelearning/hmm/PosTagger.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sequencelearning/hmm/PosTagger.java Tue Apr 12 18:55:25 2011
@@ -17,11 +17,8 @@
package org.apache.mahout.classifier.sequencelearning.hmm;
-import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.net.URL;
-import java.net.URLConnection;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
@@ -30,7 +27,8 @@ import java.util.Map;
import java.util.regex.Pattern;
import com.google.common.base.Charsets;
-import org.apache.mahout.common.IOUtils;
+import com.google.common.io.CharStreams;
+import com.google.common.io.Resources;
import org.apache.mahout.math.Matrix;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -115,7 +113,6 @@ public final class PosTagger {
* @throws IOException in case data file cannot be read.
*/
private static void readFromURL(String url, boolean assignIDs) throws IOException {
- URLConnection connection = (new URL(url)).openConnection();
// initialize the data structure
hiddenSequences = new LinkedList<int[]>();
observedSequences = new LinkedList<int[]>();
@@ -125,52 +122,46 @@ public final class PosTagger {
List<Integer> observedSequence = new LinkedList<Integer>();
List<Integer> hiddenSequence = new LinkedList<Integer>();
- BufferedReader input =
- new BufferedReader(new InputStreamReader(connection.getInputStream(), Charsets.UTF_8));
- try {
- String line;
- while ((line = input.readLine()) != null) {
- if (line.isEmpty()) {
- // new sentence starts
- int[] observedSequenceArray = new int[observedSequence.size()];
- int[] hiddenSequenceArray = new int[hiddenSequence.size()];
- for (int i = 0; i < observedSequence.size(); ++i) {
- observedSequenceArray[i] = observedSequence.get(i);
- hiddenSequenceArray[i] = hiddenSequence.get(i);
- }
- // now register those arrays
- hiddenSequences.add(hiddenSequenceArray);
- observedSequences.add(observedSequenceArray);
- // and reset the linked lists
- observedSequence.clear();
- hiddenSequence.clear();
- continue;
+ for (String line : CharStreams.readLines(Resources.newReaderSupplier(new URL(url), Charsets.UTF_8))) {
+ if (line.isEmpty()) {
+ // new sentence starts
+ int[] observedSequenceArray = new int[observedSequence.size()];
+ int[] hiddenSequenceArray = new int[hiddenSequence.size()];
+ for (int i = 0; i < observedSequence.size(); ++i) {
+ observedSequenceArray[i] = observedSequence.get(i);
+ hiddenSequenceArray[i] = hiddenSequence.get(i);
}
- readLines++;
- // we expect the format [word] [POS tag] [NP tag]
- String[] tags = SPACE.split(line);
- // when analyzing the training set, assign IDs
- if (assignIDs) {
- if (!wordIDs.containsKey(tags[0])) {
- wordIDs.put(tags[0], nextWordId++);
- }
- if (!tagIDs.containsKey(tags[1])) {
- tagIDs.put(tags[1], nextTagId++);
- }
+ // now register those arrays
+ hiddenSequences.add(hiddenSequenceArray);
+ observedSequences.add(observedSequenceArray);
+ // and reset the linked lists
+ observedSequence.clear();
+ hiddenSequence.clear();
+ continue;
+ }
+ readLines++;
+ // we expect the format [word] [POS tag] [NP tag]
+ String[] tags = SPACE.split(line);
+ // when analyzing the training set, assign IDs
+ if (assignIDs) {
+ if (!wordIDs.containsKey(tags[0])) {
+ wordIDs.put(tags[0], nextWordId++);
+ }
+ if (!tagIDs.containsKey(tags[1])) {
+ tagIDs.put(tags[1], nextTagId++);
}
- // determine the IDs
- Integer wordID = wordIDs.get(tags[0]);
- Integer tagID = tagIDs.get(tags[1]);
- // handle unknown values
- wordID = (wordID == null) ? 0 : wordID;
- tagID = (tagID == null) ? 0 : tagID;
- // now construct the current sequence
- observedSequence.add(wordID);
- hiddenSequence.add(tagID);
}
- } finally {
- IOUtils.quietClose(input);
+ // determine the IDs
+ Integer wordID = wordIDs.get(tags[0]);
+ Integer tagID = tagIDs.get(tags[1]);
+ // handle unknown values
+ wordID = (wordID == null) ? 0 : wordID;
+ tagID = (tagID == null) ? 0 : tagID;
+ // now construct the current sequence
+ observedSequence.add(wordID);
+ hiddenSequence.add(tagID);
}
+
// if there is still something in the pipe, register it
if (!observedSequence.isEmpty()) {
int[] observedSequenceArray = new int[observedSequence.size()];
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/SimpleCsvExamples.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/SimpleCsvExamples.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/SimpleCsvExamples.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/SimpleCsvExamples.java Tue Apr 12 18:55:25 2011
@@ -21,6 +21,7 @@ import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
+import com.google.common.io.Files;
import org.apache.mahout.common.IOUtils;
import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.math.DenseVector;
@@ -36,7 +37,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
import java.util.List;
@@ -90,8 +90,7 @@ public final class SimpleCsvExamples {
IOUtils.quietClose(out);
}
} else if ("--parse".equals(args[0])) {
- BufferedReader in = new BufferedReader(
- new InputStreamReader(new FileInputStream(new File(args[1])), Charsets.UTF_8));
+ BufferedReader in = Files.newReader(new File(args[1]), Charsets.UTF_8);
try {
String line = in.readLine();
while (line != null) {
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/TrainNewsGroups.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/TrainNewsGroups.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/TrainNewsGroups.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/TrainNewsGroups.java Tue Apr 12 18:55:25 2011
@@ -24,6 +24,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multiset;
import com.google.common.collect.Ordering;
+import com.google.common.io.Files;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
@@ -43,9 +44,7 @@ import org.apache.mahout.vectorizer.enco
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.text.SimpleDateFormat;
@@ -284,8 +283,7 @@ public final class TrainNewsGroups {
long date = (long) (1000 * (DATE_REFERENCE + actual * MONTH + 1 * WEEK * rand.nextDouble()));
Multiset<String> words = ConcurrentHashMultiset.create();
- BufferedReader reader =
- new BufferedReader(new InputStreamReader(new FileInputStream(file), Charsets.UTF_8));
+ BufferedReader reader = Files.newReader(file, Charsets.UTF_8);
try {
String line = reader.readLine();
Reader dateString = new StringReader(DATE_FORMATS[leakType % 3].format(new Date(date)));
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplaySpectralKMeans.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplaySpectralKMeans.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplaySpectralKMeans.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplaySpectralKMeans.java Tue Apr 12 18:55:25 2011
@@ -20,11 +20,10 @@ package org.apache.mahout.clustering.dis
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.io.File;
-import java.io.FileOutputStream;
-import java.io.OutputStreamWriter;
import java.io.Writer;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -57,8 +56,7 @@ class DisplaySpectralKMeans extends Disp
if (!fs.exists(output)) {
fs.mkdirs(output);
}
- Writer writer = new OutputStreamWriter(
- new FileOutputStream(new File(affinities.toString())), Charsets.UTF_8);
+ Writer writer = Files.newWriter(new File(affinities.toString()), Charsets.UTF_8);
try {
for (int i = 0; i < SAMPLE_DATA.size(); i++) {
for (int j = 0; j < SAMPLE_DATA.size(); j++) {
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/minhash/LastfmDataConverter.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/minhash/LastfmDataConverter.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/minhash/LastfmDataConverter.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/minhash/LastfmDataConverter.java Tue Apr 12 18:55:25 2011
@@ -18,6 +18,7 @@
package org.apache.mahout.clustering.minhash;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -29,9 +30,7 @@ import org.apache.mahout.math.VectorWrit
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -100,8 +99,7 @@ public final class LastfmDataConverter {
Map<String, Integer> featureIdxMap = new HashMap<String, Integer>();
Map<String, List<Integer>> itemFeaturesMap = new HashMap<String, List<Integer>>();
String msg = usedMemory() + "Converting data to internal vector format: ";
- BufferedReader br = new BufferedReader(
- new InputStreamReader(new FileInputStream(new File(inputFile)), Charsets.UTF_8));
+ BufferedReader br = Files.newReader(new File(inputFile), Charsets.UTF_8);
try {
System.out.print(msg);
int prevPercentDone = 1;
Modified: mahout/trunk/math/src/main/java/org/apache/mahout/common/DevURandomSeedGenerator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/main/java/org/apache/mahout/common/DevURandomSeedGenerator.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/math/src/main/java/org/apache/mahout/common/DevURandomSeedGenerator.java (original)
+++ mahout/trunk/math/src/main/java/org/apache/mahout/common/DevURandomSeedGenerator.java Tue Apr 12 18:55:25 2011
@@ -37,6 +37,9 @@
package org.apache.mahout.common;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Closeables;
+import com.google.common.io.LimitInputStream;
import org.uncommons.maths.random.SeedException;
import org.uncommons.maths.random.SeedGenerator;
@@ -61,19 +64,10 @@ public final class DevURandomSeedGenerat
*/
@Override
public byte[] generateSeed(int length) throws SeedException {
- FileInputStream file = null;
+ FileInputStream in = null;
try {
- file = new FileInputStream(DEV_URANDOM);
- byte[] randomSeed = new byte[length];
- int count = 0;
- while (count < length) {
- int bytesRead = file.read(randomSeed, count, length - count);
- if (bytesRead == -1) {
- throw new SeedException("EOF encountered reading random data.");
- }
- count += bytesRead;
- }
- return randomSeed;
+ in = new FileInputStream(DEV_URANDOM);
+ return ByteStreams.toByteArray(new LimitInputStream(in, length));
} catch (IOException ex) {
throw new SeedException("Failed reading from " + DEV_URANDOM.getName(), ex);
} catch (SecurityException ex) {
@@ -81,13 +75,7 @@ public final class DevURandomSeedGenerat
// an applet sandbox).
throw new SeedException("SecurityManager prevented access to " + DEV_URANDOM.getName(), ex);
} finally {
- if (file != null) {
- try {
- file.close();
- } catch (IOException ex) {
- // Ignore.
- }
- }
+ Closeables.closeQuietly(in);
}
}
Modified: mahout/trunk/math/src/main/java/org/apache/mahout/common/RandomUtils.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/main/java/org/apache/mahout/common/RandomUtils.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/math/src/main/java/org/apache/mahout/common/RandomUtils.java (original)
+++ mahout/trunk/math/src/main/java/org/apache/mahout/common/RandomUtils.java Tue Apr 12 18:55:25 2011
@@ -22,6 +22,8 @@ import java.util.Map;
import java.util.Random;
import java.util.WeakHashMap;
+import com.google.common.primitives.Longs;
+
/**
* <p>
* The source of random stuff for the whole project. This lets us make all randomness in the project
@@ -87,8 +89,7 @@ public final class RandomUtils {
/** @return what {@link Double#hashCode()} would return for the same value */
public static int hashDouble(double value) {
- long bits = Double.doubleToLongBits(value);
- return (int) (bits ^ bits >>> 32);
+ return Longs.hashCode(Double.doubleToLongBits(value));
}
/** @return what {@link Float#hashCode()} would return for the same value */
@@ -96,10 +97,6 @@ public final class RandomUtils {
return Float.floatToIntBits(value);
}
- public static int hashLong(long value) {
- return (int) (value ^ value >>> 32);
- }
-
/**
* <p>
* Finds next-largest "twin primes": numbers p and p+2 such that both are prime. Finds the smallest such p
Modified: mahout/trunk/math/src/main/java/org/apache/mahout/common/RandomWrapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/main/java/org/apache/mahout/common/RandomWrapper.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/math/src/main/java/org/apache/mahout/common/RandomWrapper.java (original)
+++ mahout/trunk/math/src/main/java/org/apache/mahout/common/RandomWrapper.java Tue Apr 12 18:55:25 2011
@@ -17,17 +17,17 @@
package org.apache.mahout.common;
+import com.google.common.base.Charsets;
import org.uncommons.maths.random.MersenneTwisterRNG;
import org.uncommons.maths.random.RepeatableRNG;
import org.uncommons.maths.random.SeedException;
import org.uncommons.maths.random.SeedGenerator;
-import java.nio.charset.Charset;
import java.util.Random;
public final class RandomWrapper extends Random {
- private static final byte[] STANDARD_SEED = "Mahout=Hadoop+ML".getBytes(Charset.forName("US-ASCII"));
+ private static final byte[] STANDARD_SEED = "Mahout=Hadoop+ML".getBytes(Charsets.US_ASCII);
private static final SeedGenerator SEED_GENERATOR = new FastRandomSeedGenerator();
private static boolean testSeed;
Modified: mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java (original)
+++ mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java Tue Apr 12 18:55:25 2011
@@ -19,6 +19,7 @@ package org.apache.mahout.math;
import java.util.Iterator;
+import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.math.function.DoubleDoubleFunction;
import org.apache.mahout.math.function.DoubleFunction;
@@ -553,8 +554,7 @@ public abstract class AbstractVector imp
Iterator<Element> iter = iterateNonZero();
while (iter.hasNext()) {
Element ele = iter.next();
- long v = Double.doubleToLongBits(ele.get());
- result += ele.index() * (int) (v ^ (v >>> 32));
+ result += ele.index() * RandomUtils.hashDouble(ele.get());
}
return result;
}
Modified: mahout/trunk/math/src/test/java/org/apache/mahout/common/RandomUtilsTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/test/java/org/apache/mahout/common/RandomUtilsTest.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/math/src/test/java/org/apache/mahout/common/RandomUtilsTest.java (original)
+++ mahout/trunk/math/src/test/java/org/apache/mahout/common/RandomUtilsTest.java Tue Apr 12 18:55:25 2011
@@ -28,6 +28,22 @@ import java.util.Random;
public final class RandomUtilsTest extends MahoutTestCase {
@Test
+ public void testHashDouble() {
+ assertEquals(new Double(0.0).hashCode(), RandomUtils.hashDouble(0.0));
+ assertEquals(new Double(1.0).hashCode(), RandomUtils.hashDouble(1.0));
+ assertEquals(new Double(Double.POSITIVE_INFINITY).hashCode(), RandomUtils.hashDouble(Double.POSITIVE_INFINITY));
+ assertEquals(new Double(Double.NaN).hashCode(), RandomUtils.hashDouble(Double.NaN));
+ }
+
+ @Test
+ public void testHashFloat() {
+ assertEquals(new Float(0.0f).hashCode(), RandomUtils.hashFloat(0.0f));
+ assertEquals(new Float(1.0f).hashCode(), RandomUtils.hashFloat(1.0f));
+ assertEquals(new Float(Float.POSITIVE_INFINITY).hashCode(), RandomUtils.hashFloat(Float.POSITIVE_INFINITY));
+ assertEquals(new Float(Float.NaN).hashCode(), RandomUtils.hashFloat(Float.NaN));
+ }
+
+ @Test
public void testIsNotPrime() {
assertTrue(RandomUtils.isNotPrime(Integer.MIN_VALUE));
assertTrue(RandomUtils.isNotPrime(-1));
@@ -83,31 +99,4 @@ public final class RandomUtilsTest exten
}
}
- @Test
- public void testHashDouble() {
- assertEquals(0, RandomUtils.hashDouble(0.0));
- assertEquals(1072693248, RandomUtils.hashDouble(1.0));
- assertEquals(2146959360, RandomUtils.hashDouble(Double.NaN));
- assertEquals(2146435072, RandomUtils.hashDouble(Double.POSITIVE_INFINITY));
- }
-
- @Test
- public void testHashFloat() {
- assertEquals(0, RandomUtils.hashFloat(0.0f));
- assertEquals(1065353216, RandomUtils.hashFloat(1.0f));
- assertEquals(2143289344, RandomUtils.hashFloat(Float.NaN));
- assertEquals(2139095040, RandomUtils.hashFloat(Float.POSITIVE_INFINITY));
- }
-
- @Test
- public void testHashLong() {
- assertEquals(0, RandomUtils.hashLong(-1L));
- assertEquals(0, RandomUtils.hashLong(0L));
- assertEquals(1, RandomUtils.hashLong(1L));
- assertEquals(Integer.MAX_VALUE, RandomUtils.hashLong(Integer.MAX_VALUE));
- assertEquals(Integer.MIN_VALUE, RandomUtils.hashLong((long) Integer.MAX_VALUE + 1L));
- assertEquals(Integer.MIN_VALUE, RandomUtils.hashLong(Long.MAX_VALUE));
- assertEquals(Integer.MIN_VALUE, RandomUtils.hashLong(Long.MIN_VALUE));
- }
-
}
Modified: mahout/trunk/taste-web/src/main/java/org/apache/mahout/cf/taste/web/RecommenderWrapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/taste-web/src/main/java/org/apache/mahout/cf/taste/web/RecommenderWrapper.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/taste-web/src/main/java/org/apache/mahout/cf/taste/web/RecommenderWrapper.java (original)
+++ mahout/trunk/taste-web/src/main/java/org/apache/mahout/cf/taste/web/RecommenderWrapper.java Tue Apr 12 18:55:25 2011
@@ -17,22 +17,22 @@
package org.apache.mahout.cf.taste.web;
+import com.google.common.io.Files;
+import com.google.common.io.InputSupplier;
+import com.google.common.io.Resources;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.recommender.IDRescorer;
import org.apache.mahout.cf.taste.recommender.RecommendedItem;
import org.apache.mahout.cf.taste.recommender.Recommender;
-import org.apache.mahout.common.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
+import java.net.URL;
import java.util.Collection;
import java.util.List;
@@ -108,30 +108,19 @@ public abstract class RecommenderWrapper
public static File readResourceToTempFile(String resourceName) throws IOException {
String absoluteResource = resourceName.startsWith("/") ? resourceName : '/' + resourceName;
log.info("Loading resource {}", absoluteResource);
- InputStream is = RecommenderWrapper.class.getResourceAsStream(absoluteResource);
- if (is == null) {
+ InputSupplier<? extends InputStream> inSupplier;
+ try {
+ URL resourceURL = Resources.getResource(RecommenderWrapper.class, absoluteResource);
+ inSupplier = Resources.newInputStreamSupplier(resourceURL);
+ } catch (IllegalArgumentException iae) {
File resourceFile = new File(resourceName);
log.info("Falling back to load file {}", resourceFile.getAbsolutePath());
- is = new FileInputStream(resourceFile);
- }
- try {
- File tempFile = File.createTempFile("taste", null);
- tempFile.deleteOnExit();
- OutputStream os = new FileOutputStream(tempFile);
- try {
- int bytesRead;
- byte[] buffer = new byte[32768];
- while ((bytesRead = is.read(buffer)) > 0) {
- os.write(buffer, 0, bytesRead);
- }
- os.flush();
- return tempFile;
- } finally {
- IOUtils.quietClose(os);
- }
- } finally {
- IOUtils.quietClose(is);
+ inSupplier = Files.newInputStreamSupplier(resourceFile);
}
+ File tempFile = File.createTempFile("taste", null);
+ tempFile.deleteOnExit();
+ Files.copy(inSupplier, tempFile);
+ return tempFile;
}
}
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/lda/LDAPrintTopics.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/lda/LDAPrintTopics.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/lda/LDAPrintTopics.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/lda/LDAPrintTopics.java Tue Apr 12 18:55:25 2011
@@ -18,9 +18,7 @@
package org.apache.mahout.clustering.lda;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
@@ -31,6 +29,7 @@ import java.util.PriorityQueue;
import java.util.Queue;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
import org.apache.commons.cli2.Option;
@@ -228,8 +227,7 @@ public final class LDAPrintTopics {
private static void writeTopWords(List<List<String>> topWords, File output) throws IOException {
for (int i = 0; i < topWords.size(); ++i) {
List<String> topK = topWords.get(i);
- Writer writer = new OutputStreamWriter(
- new FileOutputStream(new File(output, "topic-" + i)), Charsets.UTF_8);
+ Writer writer = Files.newWriter(new File(output, "topic-" + i), Charsets.UTF_8);
try {
writer.write("Topic " + i + '\n');
writer.write("===========\n");
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java Tue Apr 12 18:55:25 2011
@@ -18,11 +18,11 @@
package org.apache.mahout.utils;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
import org.apache.commons.cli2.Option;
@@ -85,8 +85,7 @@ public final class SequenceFileDumper {
Writer writer;
if (cmdLine.hasOption(outputOpt)) {
- writer = new OutputStreamWriter(
- new FileOutputStream(new File(cmdLine.getValue(outputOpt).toString())), Charsets.UTF_8);
+ writer = Files.newWriter(new File(cmdLine.getValue(outputOpt).toString()), Charsets.UTF_8);
} else {
writer = new OutputStreamWriter(System.out);
}
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java Tue Apr 12 18:55:25 2011
@@ -18,7 +18,6 @@
package org.apache.mahout.utils.clustering;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
@@ -33,6 +32,7 @@ import java.util.Map;
import java.util.TreeMap;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -139,7 +139,7 @@ public final class ClusterDumper extends
if (this.outputFile == null) {
writer = new OutputStreamWriter(System.out);
} else {
- writer = new OutputStreamWriter(new FileOutputStream(new File(this.outputFile)), Charsets.UTF_8);
+ writer = Files.newWriter(new File(this.outputFile), Charsets.UTF_8);
}
try {
for (Cluster value :
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java Tue Apr 12 18:55:25 2011
@@ -18,6 +18,7 @@
package org.apache.mahout.utils.vectors;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
import org.apache.commons.cli2.Option;
@@ -40,7 +41,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
@@ -131,8 +131,7 @@ public final class VectorDumper {
boolean transposeKeyValue = cmdLine.hasOption(vectorAsKeyOpt);
Writer writer;
if (cmdLine.hasOption(outputOpt)) {
- writer = new OutputStreamWriter(
- new FileOutputStream(new File(cmdLine.getValue(outputOpt).toString())), Charsets.UTF_8);
+ writer = Files.newWriter(new File(cmdLine.getValue(outputOpt).toString()), Charsets.UTF_8);
} else {
writer = new OutputStreamWriter(System.out);
}
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java Tue Apr 12 18:55:25 2011
@@ -19,9 +19,7 @@ package org.apache.mahout.utils.vectors.
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.nio.charset.Charset;
@@ -32,6 +30,7 @@ import java.util.Locale;
import java.util.regex.Pattern;
import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.mahout.math.Vector;
/**
@@ -61,7 +60,7 @@ public class ARFFVectorIterable implemen
}
public ARFFVectorIterable(File file, Charset encoding, ARFFModel model) throws IOException {
- this(new InputStreamReader(new FileInputStream(file), encoding), model);
+ this(Files.newReader(file, encoding), model);
}
public ARFFVectorIterable(String arff, ARFFModel model) throws IOException {
@@ -138,7 +137,6 @@ public class ARFFVectorIterable implemen
}
-
@Override
public Iterator<Vector> iterator() {
return new ARFFIterator(buff, model);
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java?rev=1091538&r1=1091537&r2=1091538&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java Tue Apr 12 18:55:25 2011
@@ -17,16 +17,14 @@
package org.apache.mahout.utils.vectors.arff;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
-import java.io.OutputStreamWriter;
import java.io.Writer;
-import java.nio.charset.Charset;
import java.util.Map;
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
import org.apache.commons.cli2.Option;
@@ -130,8 +128,7 @@ public final class Driver {
}
log.info("Dictionary Output file: {}", dictOut);
Map<String,Integer> labels = model.getLabelBindings();
- Writer writer = new BufferedWriter(
- new OutputStreamWriter(new FileOutputStream(dictOut), Charset.forName("UTF8")));
+ Writer writer = Files.newWriter(dictOut, Charsets.UTF_8);
try {
for (Map.Entry<String,Integer> entry : labels.entrySet()) {
writer.write(entry.getKey());