You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2010/06/02 14:37:38 UTC
svn commit: r950537 [2/2] - in /mahout/trunk: buildtools/src/main/resources/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/
core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/
core/src/main/java/org/apache/mahout/cf/taste/i...
Modified: mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java (original)
+++ mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java Wed Jun 2 12:37:36 2010
@@ -19,12 +19,10 @@ package org.apache.mahout.math;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
-import com.google.gson.reflect.TypeToken;
import org.apache.mahout.math.function.BinaryFunction;
import org.apache.mahout.math.function.PlusMult;
import org.apache.mahout.math.function.UnaryFunction;
-import java.lang.reflect.Type;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -168,27 +166,19 @@ public abstract class AbstractMatrix imp
}
public static Matrix decodeMatrix(String formatString) {
- Type vectorType = new TypeToken<Vector>() {
- }.getType();
- Type matrixType = new TypeToken<Matrix>() {
- }.getType();
GsonBuilder builder = new GsonBuilder();
- builder.registerTypeAdapter(vectorType, new JsonVectorAdapter());
- builder.registerTypeAdapter(matrixType, new JsonMatrixAdapter());
+ builder.registerTypeAdapter(Vector.class, new JsonVectorAdapter());
+ builder.registerTypeAdapter(Matrix.class, new JsonMatrixAdapter());
Gson gson = builder.create();
- return gson.fromJson(formatString, matrixType);
+ return gson.fromJson(formatString, Matrix.class);
}
public String asFormatString() {
- Type vectorType = new TypeToken<Vector>() {
- }.getType();
- Type matrixType = new TypeToken<Matrix>() {
- }.getType();
GsonBuilder builder = new GsonBuilder();
- builder.registerTypeAdapter(vectorType, new JsonVectorAdapter());
- builder.registerTypeAdapter(matrixType, new JsonMatrixAdapter());
+ builder.registerTypeAdapter(Vector.class, new JsonVectorAdapter());
+ builder.registerTypeAdapter(Matrix.class, new JsonMatrixAdapter());
Gson gson = builder.create();
- return gson.toJson(this, matrixType);
+ return gson.toJson(this, Matrix.class);
}
public Matrix assign(double value) {
Modified: mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java (original)
+++ mahout/trunk/math/src/main/java/org/apache/mahout/math/AbstractVector.java Wed Jun 2 12:37:36 2010
@@ -17,7 +17,6 @@
package org.apache.mahout.math;
-import java.lang.reflect.Type;
import java.util.Iterator;
import org.apache.mahout.math.function.BinaryFunction;
@@ -25,7 +24,6 @@ import org.apache.mahout.math.function.U
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
-import com.google.gson.reflect.TypeToken;
/** Implementations of generic capabilities like sum of elements and dot products */
public abstract class AbstractVector implements Vector {
@@ -459,12 +457,10 @@ public abstract class AbstractVector imp
* @return the n-dimensional point
*/
public static Vector decodeVector(String formattedString) {
- Type vectorType = new TypeToken<Vector>() {
- }.getType();
GsonBuilder builder = new GsonBuilder();
- builder.registerTypeAdapter(vectorType, new JsonVectorAdapter());
+ builder.registerTypeAdapter(Vector.class, new JsonVectorAdapter());
Gson gson = builder.create();
- return gson.fromJson(formattedString, vectorType);
+ return gson.fromJson(formattedString, Vector.class);
}
public final int size() {
@@ -472,12 +468,10 @@ public abstract class AbstractVector imp
}
public String asFormatString() {
- Type vectorType = new TypeToken<Vector>() {
- }.getType();
GsonBuilder builder = new GsonBuilder();
- builder.registerTypeAdapter(vectorType, new JsonVectorAdapter());
+ builder.registerTypeAdapter(Vector.class, new JsonVectorAdapter());
Gson gson = builder.create();
- return gson.toJson(this, vectorType);
+ return gson.toJson(this, Vector.class);
}
@Override
Modified: mahout/trunk/math/src/main/java/org/apache/mahout/math/JsonMatrixAdapter.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/main/java/org/apache/mahout/math/JsonMatrixAdapter.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/math/src/main/java/org/apache/mahout/math/JsonMatrixAdapter.java (original)
+++ mahout/trunk/math/src/main/java/org/apache/mahout/math/JsonMatrixAdapter.java Wed Jun 2 12:37:36 2010
@@ -30,21 +30,16 @@ import com.google.gson.reflect.TypeToken
import java.lang.reflect.Type;
-public class JsonMatrixAdapter implements JsonSerializer<Matrix>,
- JsonDeserializer<Matrix> {
+public class JsonMatrixAdapter implements JsonSerializer<Matrix>, JsonDeserializer<Matrix> {
public static final String CLASS = "class";
public static final String MATRIX = "matrix";
public JsonElement serialize(Matrix src, Type typeOfSrc,
JsonSerializationContext context) {
- Type vectorType = new TypeToken<Vector>() {
- }.getType();
- Type matrixType = new TypeToken<Matrix>() {
- }.getType();
GsonBuilder builder = new GsonBuilder();
- builder.registerTypeAdapter(vectorType, new JsonVectorAdapter());
- builder.registerTypeAdapter(matrixType, new JsonMatrixAdapter());
+ builder.registerTypeAdapter(Vector.class, new JsonVectorAdapter());
+ builder.registerTypeAdapter(Matrix.class, new JsonMatrixAdapter());
Gson gson = builder.create();
JsonObject obj = new JsonObject();
obj.add(CLASS, new JsonPrimitive(src.getClass().getName()));
@@ -54,19 +49,15 @@ public class JsonMatrixAdapter implement
public Matrix deserialize(JsonElement json, Type typeOfT,
JsonDeserializationContext context) throws JsonParseException {
- Type vectorType = new TypeToken<Vector>() {
- }.getType();
- Type matrixType = new TypeToken<Matrix>() {
- }.getType();
GsonBuilder builder = new GsonBuilder();
- builder.registerTypeAdapter(vectorType, new JsonVectorAdapter());
- builder.registerTypeAdapter(matrixType, new JsonMatrixAdapter());
+ builder.registerTypeAdapter(Vector.class, new JsonVectorAdapter());
+ builder.registerTypeAdapter(Matrix.class, new JsonMatrixAdapter());
Gson gson = builder.create();
JsonObject obj = json.getAsJsonObject();
String klass = obj.get(CLASS).getAsString();
String matrix = obj.get(MATRIX).getAsString();
ClassLoader ccl = Thread.currentThread().getContextClassLoader();
- Class<?> cl = null;
+ Class<?> cl;
try {
cl = ccl.loadClass(klass);
} catch (ClassNotFoundException e) {
Modified: mahout/trunk/maven/src/main/resources/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/mahout/trunk/maven/src/main/resources/findbugs-exclude.xml?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/maven/src/main/resources/findbugs-exclude.xml (original)
+++ mahout/trunk/maven/src/main/resources/findbugs-exclude.xml Wed Jun 2 12:37:36 2010
@@ -1,12 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<FindBugsFilter>
<Match>
+ <Bug pattern="DLS_DEAD_LOCAL_STORE"/>
+ </Match>
+ <Match>
+ <Bug pattern="DLS_DEAD_LOCAL_STORE_OF_NULL"/>
+ </Match>
+ <Match>
<Bug pattern="SE_NO_SERIALVERSIONID"/>
</Match>
<Match>
<Bug pattern="EI_EXPOSE_REP2"/>
</Match>
<Match>
+ <Bug pattern="SIC_INNER_SHOULD_BE_STATIC_ANON"/>
+ </Match>
+ <Match>
<Bug pattern="SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING"/>
</Match>
<Match>
@@ -16,7 +25,10 @@
<Bug pattern="SE_BAD_FIELD"/>
</Match>
<Match>
- <Bug pattern="RV_RETURN_VALUE_IGNORE"/>
+ <Bug pattern="RV_RETURN_VALUE_IGNORED"/>
+ </Match>
+ <Match>
+ <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"/>
</Match>
<Match>
<Bug pattern="UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR"/>
Modified: mahout/trunk/maven/src/main/resources/mahout-checkstyle.xml
URL: http://svn.apache.org/viewvc/mahout/trunk/maven/src/main/resources/mahout-checkstyle.xml?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/maven/src/main/resources/mahout-checkstyle.xml (original)
+++ mahout/trunk/maven/src/main/resources/mahout-checkstyle.xml Wed Jun 2 12:37:36 2010
@@ -164,11 +164,11 @@
<module name="IllegalInstantiation" />
<!--<module name="IllegalToken"/>-->
<!--<module name="IllegalTokenText"/>-->
- <module name="InnerAssignment" />
+ <!--<module name="InnerAssignment"/>-->
<!--<module name="MagicNumber"/>-->
<module name="MissingSwitchDefault" />
<!--module name="ModifiedControlVariable"/-->
- <!--<module name="SimplifyBooleanExpression" /> -->
+ <module name="SimplifyBooleanExpression" />
<module name="SimplifyBooleanReturn" />
<module name="StringLiteralEquality" />
<module name="NestedIfDepth">
@@ -184,7 +184,7 @@
<property name="illegalClassNames"
value="java.lang.Error,java.lang.RuntimeException" />
</module>
- <!--<module name="RedundantThrows"/>-->
+ <module name="RedundantThrows"/>
<module name="PackageDeclaration" />
<!--module name="JUnitTestCase" /-->
<module name="ReturnCount">
@@ -196,7 +196,7 @@
<property name="illegalClassNames"
value="java.util.GregorianCalendar, java.util.Hashtable, java.util.HashSet, java.util.HashMap, java.util.ArrayList, java.util.LinkedList, java.util.LinkedHashMap, java.util.LinkedHashSet, java.util.TreeSet, java.util.TreeMap" />
</module>
- <module name="DeclarationOrder" />
+ <!--module name="DeclarationOrder" /-->
<!--<module name="ParameterAssignment"/>-->
<module name="ExplicitInitialization" />
<module name="DefaultComesLast" />
@@ -215,7 +215,7 @@
<module name="FinalClass" />
<module name="HideUtilityClassConstructor" />
<module name="InterfaceIsType" />
- <!--<module name="MutableException"/>-->
+ <module name="MutableException"/>
<module name="ThrowsCount">
<property name="max" value="5" />
</module>
@@ -246,8 +246,8 @@
<!-- Miscellaneous other checks. -->
<!-- See http://checkstyle.sf.net/config_misc.html -->
+ <module name="ArrayTypeStyle"/>
<!--
- <module name="ArrayTypeStyle"/>
<module name="FinalParameters"/>
-->
<!--
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwDriver.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwDriver.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwDriver.java Wed Jun 2 12:37:36 2010
@@ -17,7 +17,6 @@
package org.apache.mahout.clustering.cdbw;
-import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
@@ -43,7 +42,6 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.mahout.clustering.Cluster;
-import org.apache.mahout.clustering.ClusterBase;
import org.apache.mahout.clustering.WeightedVectorWritable;
import org.apache.mahout.clustering.dirichlet.DirichletCluster;
import org.apache.mahout.clustering.kmeans.KMeansDriver;
@@ -78,14 +76,15 @@ public final class CDbwDriver {
Option modelOpt = obuilder.withLongName("modelClass").withRequired(true).withShortName("d").withArgument(
abuilder.withName("modelClass").withMinimum(1).withMaximum(1).create()).withDescription(
- "The ModelDistribution class name. " + "Defaults to org.apache.mahout.clustering.dirichlet.models.NormalModelDistribution")
- .create();
+ "The ModelDistribution class name. "
+ + "Defaults to org.apache.mahout.clustering.dirichlet.models.NormalModelDistribution").create();
Option numRedOpt = obuilder.withLongName("maxRed").withRequired(true).withShortName("r").withArgument(
- abuilder.withName("maxRed").withMinimum(1).withMaximum(1).create()).withDescription("The number of reduce tasks.").create();
+ abuilder.withName("maxRed").withMinimum(1).withMaximum(1).create())
+ .withDescription("The number of reduce tasks.").create();
- Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(outputOpt).withOption(modelOpt).withOption(
- maxIterOpt).withOption(helpOpt).withOption(numRedOpt).create();
+ Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(outputOpt)
+ .withOption(modelOpt).withOption(maxIterOpt).withOption(helpOpt).withOption(numRedOpt).create();
try {
Parser parser = new Parser();
@@ -127,9 +126,15 @@ public final class CDbwDriver {
* @param numReducers
* the number of Reducers desired
*/
- public static void runJob(Path clustersIn, Path clusteredPointsIn, Path output, String distanceMeasureClass,
- int numIterations, int numReducers) throws ClassNotFoundException, InstantiationException, IllegalAccessException,
- IOException, SecurityException, NoSuchMethodException, InvocationTargetException {
+ public static void runJob(Path clustersIn,
+ Path clusteredPointsIn,
+ Path output,
+ String distanceMeasureClass,
+ int numIterations,
+ int numReducers) throws ClassNotFoundException,
+ InstantiationException,
+ IllegalAccessException,
+ IOException {
Path stateIn = new Path(output, "representativePoints-0");
writeInitialState(stateIn, clustersIn);
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwEvaluator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwEvaluator.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwEvaluator.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwEvaluator.java Wed Jun 2 12:37:36 2010
@@ -36,13 +36,13 @@ import org.apache.mahout.math.function.S
public class CDbwEvaluator {
- Map<Integer, List<VectorWritable>> representativePoints;
+ private final Map<Integer, List<VectorWritable>> representativePoints;
- Map<Integer, Double> stDevs = new HashMap<Integer, Double>();
+ private final Map<Integer, Double> stDevs = new HashMap<Integer, Double>();
- Map<Integer, Cluster> clusters;
+ private final Map<Integer, Cluster> clusters;
- DistanceMeasure measure;
+ private final DistanceMeasure measure;
/**
* For testing only
@@ -54,8 +54,9 @@ public class CDbwEvaluator {
* @param measure
* an appropriate DistanceMeasure
*/
- public CDbwEvaluator(Map<Integer, List<VectorWritable>> representativePoints, Map<Integer, Cluster> clusters,
- DistanceMeasure measure) {
+ public CDbwEvaluator(Map<Integer, List<VectorWritable>> representativePoints,
+ Map<Integer, Cluster> clusters,
+ DistanceMeasure measure) {
this.representativePoints = representativePoints;
this.clusters = clusters;
this.measure = measure;
@@ -73,8 +74,7 @@ public class CDbwEvaluator {
* a String path to the input clusters directory
*/
public CDbwEvaluator(JobConf job, Path clustersIn)
- throws SecurityException, IllegalArgumentException,
- ClassNotFoundException, InstantiationException, IllegalAccessException, IOException {
+ throws ClassNotFoundException, InstantiationException, IllegalAccessException, IOException {
ClassLoader ccl = Thread.currentThread().getContextClassLoader();
Class<?> cl = ccl.loadClass(job.get(CDbwDriver.DISTANCE_MEASURE_KEY));
measure = (DistanceMeasure) cl.newInstance();
@@ -96,8 +96,8 @@ public class CDbwEvaluator {
* a String pathname to the directory containing input cluster files
* @return a List<Cluster> of the clusters
*/
- private Map<Integer, Cluster> loadClusters(JobConf job, Path clustersIn)
- throws InstantiationException, IllegalAccessException, IOException, SecurityException {
+ private static Map<Integer, Cluster> loadClusters(JobConf job, Path clustersIn)
+ throws InstantiationException, IllegalAccessException, IOException {
Map<Integer, Cluster> clusters = new HashMap<Integer, Cluster>();
FileSystem fs = clustersIn.getFileSystem(job);
for (FileStatus part : fs.listStatus(clustersIn)) {
@@ -118,14 +118,17 @@ public class CDbwEvaluator {
}
double interClusterDensity() {
- double sum = 0;
- for (int cI : representativePoints.keySet()) {
- for (int cJ : representativePoints.keySet()) {
- if (cI == cJ) {
+ double sum = 0.0;
+ for (Map.Entry<Integer, List<VectorWritable>> entry1 : representativePoints.entrySet()) {
+ Integer cI = entry1.getKey();
+ List<VectorWritable> repI = entry1.getValue();
+ double stDevI = stDevs.get(cI);
+ for (Map.Entry<Integer, List<VectorWritable>> entry2 : representativePoints.entrySet()) {
+ Integer cJ = entry2.getKey();
+ if (cI.equals(cJ)) {
continue;
}
- List<VectorWritable> repI = representativePoints.get(cI);
- List<VectorWritable> repJ = representativePoints.get(cJ);
+ List<VectorWritable> repJ = entry2.getValue();
double minDistance = Double.MAX_VALUE;
Vector uIJ = null;
for (VectorWritable aRepI : repI) {
@@ -139,12 +142,11 @@ public class CDbwEvaluator {
}
}
}
- double stDevI = stDevs.get(cI);
double stDevJ = stDevs.get(cJ);
double interDensity = interDensity(uIJ, cI, cJ);
double stdSum = stDevI + stDevJ;
- double density = 0;
- if (stdSum > 0) {
+ double density = 0.0;
+ if (stdSum > 0.0) {
density = minDistance * interDensity / stdSum;
}
@@ -167,8 +169,8 @@ public class CDbwEvaluator {
double interDensity(Vector uIJ, int cI, int cJ) {
List<VectorWritable> repI = representativePoints.get(cI);
List<VectorWritable> repJ = representativePoints.get(cJ);
- double density = 0;
- double std = (stDevs.get(cI) + stDevs.get(cJ)) / 2;
+ double density = 0.0;
+ double std = (stDevs.get(cI) + stDevs.get(cJ)) / 2.0;
for (VectorWritable vwI : repI) {
if (measure.distance(uIJ, vwI.get()) <= std) {
density++;
@@ -193,16 +195,8 @@ public class CDbwEvaluator {
for (VectorWritable vw : repPts) {
s0++;
Vector v = vw.get();
- if (s1 == null) {
- s1 = v.clone();
- } else {
- s1 = s1.plus(v);
- }
- if (s2 == null) {
- s2 = v.times(v);
- } else {
- s2 = s2.plus(v.times(v));
- }
+ s1 = s1 == null ? v.clone() : s1.plus(v);
+ s2 = s2 == null ? v.times(v) : s2.plus(v.times(v));
}
Vector std = s2.times(s0).minus(s1.times(s1)).assign(new SquareRootFunction()).divide(s0);
double d = std.zSum() / std.size();
@@ -210,11 +204,11 @@ public class CDbwEvaluator {
stDevs.put(cI, d);
}
- double minRpDistance(List<VectorWritable> repI, List<VectorWritable> repJ) {
+ double minRpDistance(Iterable<VectorWritable> repI, Iterable<VectorWritable> repJ) {
double minDistance = Double.MAX_VALUE;
- for (int ptI = 0; ptI < repI.size(); ptI++) {
- for (int ptJ = 0; ptJ < repJ.size(); ptJ++) {
- double distance = measure.distance(repI.get(ptI).get(), repJ.get(ptJ).get());
+ for (VectorWritable aRepI : repI) {
+ for (VectorWritable aRepJ : repJ) {
+ double distance = measure.distance(aRepI.get(), aRepJ.get());
if (distance < minDistance) {
minDistance = distance;
}
@@ -225,13 +219,14 @@ public class CDbwEvaluator {
double separation() {
double minDistance = Double.MAX_VALUE;
- for (Integer cI : representativePoints.keySet()) {
- for (int cJ : representativePoints.keySet()) {
- if (cI == cJ) {
+ for (Map.Entry<Integer, List<VectorWritable>> entry1 : representativePoints.entrySet()) {
+ Integer cI = entry1.getKey();
+ List<VectorWritable> repI = entry1.getValue();
+ for (Map.Entry<Integer, List<VectorWritable>> entry2 : representativePoints.entrySet()) {
+ if (cI.equals(entry2.getKey())) {
continue;
}
- List<VectorWritable> repI = representativePoints.get(cI);
- List<VectorWritable> repJ = representativePoints.get(cJ);
+ List<VectorWritable> repJ = entry2.getValue();
for (VectorWritable aRepI : repI) {
for (VectorWritable aRepJ : repJ) {
double distance = measure.distance(aRepI.get(), aRepJ.get());
@@ -242,40 +237,34 @@ public class CDbwEvaluator {
}
}
}
- double separation = minDistance / (1 + interClusterDensity());
- //System.out.println("separation=" + separation);
- return separation;
+ return minDistance / (1.0 + interClusterDensity());
}
double intraClusterDensity() {
- double avgStd = 0;
+ double avgStd = 0.0;
for (Integer cId : representativePoints.keySet()) {
avgStd += stDevs.get(cId);
}
avgStd /= representativePoints.size();
- double sum = 0;
- for (Integer cId : representativePoints.keySet()) {
- List<VectorWritable> repI = representativePoints.get(cId);
- double cSum = 0;
- for (int ptI = 0; ptI < repI.size(); ptI++) {
- double inDensity = intraDensity(clusters.get(cId).getCenter(), repI.get(ptI).get(), avgStd);
- Double std = stDevs.get(cId);
- if (std > 0) {
+ double sum = 0.0;
+ for (Map.Entry<Integer, List<VectorWritable>> entry : representativePoints.entrySet()) {
+ Integer cId = entry.getKey();
+ List<VectorWritable> repI = entry.getValue();
+ double cSum = 0.0;
+ for (VectorWritable aRepI : repI) {
+ double inDensity = intraDensity(clusters.get(cId).getCenter(), aRepI.get(), avgStd);
+ double std = stDevs.get(cId);
+ if (std > 0.0) {
cSum += inDensity / std;
}
}
sum += cSum / repI.size();
}
- double intraClusterDensity = sum / representativePoints.size();
- //System.out.println("intraClusterDensity=" + intraClusterDensity);
- return intraClusterDensity;
+ return sum / representativePoints.size();
}
double intraDensity(Vector clusterCenter, Vector repPoint, double avgStd) {
- if (measure.distance(clusterCenter, repPoint) <= avgStd) {
- return 1;
- }
- return 0;
+ return measure.distance(clusterCenter, repPoint) <= avgStd ? 1.0 : 0.0;
}
}
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwMapper.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwMapper.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwMapper.java Wed Jun 2 12:37:36 2010
@@ -48,7 +48,7 @@ public class CDbwMapper extends MapReduc
private DistanceMeasure measure = new EuclideanDistanceMeasure();
- private OutputCollector<IntWritable, WeightedVectorWritable> output = null;
+ private OutputCollector<IntWritable, WeightedVectorWritable> output;
@Override
public void map(IntWritable clusterId,
@@ -77,8 +77,7 @@ public class CDbwMapper extends MapReduc
this.measure = measure;
}
- public static Map<Integer, List<VectorWritable>> getRepresentativePoints(JobConf job)
- throws SecurityException, IllegalArgumentException {
+ public static Map<Integer, List<VectorWritable>> getRepresentativePoints(JobConf job) {
String statePath = job.get(CDbwDriver.STATE_IN_KEY);
Map<Integer, List<VectorWritable>> representativePoints = new HashMap<Integer, List<VectorWritable>>();
try {
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/text/DefaultAnalyzer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/text/DefaultAnalyzer.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/text/DefaultAnalyzer.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/text/DefaultAnalyzer.java Wed Jun 2 12:37:36 2010
@@ -19,13 +19,16 @@ package org.apache.mahout.text;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.util.Version;
-/** A subclass of the Lucene StandardAnalyzer that provides a no-argument constructor.
+/**
+ * A subclass of the Lucene StandardAnalyzer that provides a no-argument constructor.
* Used as the default analyzer in many cases where an analyzer is instantiated by
* class name by calling a no-arg constructor.
*/
public class DefaultAnalyzer extends StandardAnalyzer {
- @SuppressWarnings("deprecation")
- public DefaultAnalyzer() {
- super(Version.LUCENE_CURRENT);
- }
+
+ @SuppressWarnings("deprecation")
+ public DefaultAnalyzer() {
+ super(Version.LUCENE_CURRENT);
+ }
+
}
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/text/SparseVectorsFromSequenceFiles.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/text/SparseVectorsFromSequenceFiles.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/text/SparseVectorsFromSequenceFiles.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/text/SparseVectorsFromSequenceFiles.java Wed Jun 2 12:37:36 2010
@@ -39,13 +39,13 @@ import org.slf4j.LoggerFactory;
/**
* Converts a given set of sequence files into SparseVectors
- *
*/
public final class SparseVectorsFromSequenceFiles {
private static final Logger log = LoggerFactory.getLogger(SparseVectorsFromSequenceFiles.class);
- private SparseVectorsFromSequenceFiles() {}
+ private SparseVectorsFromSequenceFiles() {
+ }
public static void main(String[] args) throws Exception {
DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
@@ -170,7 +170,7 @@ public final class SparseVectorsFromSequ
Class<? extends Analyzer> analyzerClass = DefaultAnalyzer.class;
if (cmdLine.hasOption(analyzerNameOpt)) {
String className = cmdLine.getValue(analyzerNameOpt).toString();
- analyzerClass = (Class<? extends Analyzer>) Class.forName(className);
+ analyzerClass = Class.forName(className).asSubclass(Analyzer.class);
// try instantiating it, b/c there isn't any point in setting it if
// you can't instantiate it
analyzerClass.newInstance();
@@ -180,9 +180,9 @@ public final class SparseVectorsFromSequ
if (cmdLine.hasOption(weightOpt)) {
String wString = cmdLine.getValue(weightOpt).toString();
- if (wString.equalsIgnoreCase("tf")) {
+ if ("tf".equalsIgnoreCase(wString)) {
processIdf = false;
- } else if (wString.equalsIgnoreCase("tfidf")) {
+ } else if ("tfidf".equalsIgnoreCase(wString)) {
processIdf = true;
} else {
throw new OptionException(weightOpt);
@@ -203,7 +203,7 @@ public final class SparseVectorsFromSequ
float norm = PartialVectorMerger.NO_NORMALIZING;
if (cmdLine.hasOption(powerOpt)) {
String power = cmdLine.getValue(powerOpt).toString();
- if (power.equals("INF")) {
+ if ("INF".equals(power)) {
norm = Float.POSITIVE_INFINITY;
} else {
norm = Float.parseFloat(power);
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java Wed Jun 2 12:37:36 2010
@@ -78,9 +78,9 @@ public final class ClusterDumper {
private int numTopFeatures = 10;
- private Map<Integer, List<WeightedVectorWritable>> clusterIdToPoints = null;
+ private Map<Integer, List<WeightedVectorWritable>> clusterIdToPoints;
- private boolean useJSON = false;
+ private boolean useJSON;
public ClusterDumper(Path seqFileDir, Path pointsDir) throws IOException {
this.seqFileDir = seqFileDir;
@@ -115,51 +115,54 @@ public final class ClusterDumper {
}
Writer writer = this.outputFile == null ? new OutputStreamWriter(System.out) : new FileWriter(this.outputFile);
+ try {
+ FileSystem fs = seqFileDir.getFileSystem(conf);
+ for (FileStatus seqFile : fs.globStatus(new Path(seqFileDir, "part-*"))) {
+ Path path = seqFile.getPath();
+ //System.out.println("Input Path: " + path); doesn't this interfere with output?
+ SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
+ try {
+ Writable key = (Writable) reader.getKeyClass().newInstance();
+ Writable value = (Writable) reader.getValueClass().newInstance();
+ while (reader.next(key, value)) {
+ Cluster cluster = (Cluster) value;
+ String fmtStr = useJSON ? cluster.asJsonString() : cluster.asFormatString(dictionary);
+ if (subString > 0 && fmtStr.length() > subString) {
+ writer.write(':');
+ writer.write(fmtStr, 0, Math.min(subString, fmtStr.length()));
+ } else {
+ writer.write(fmtStr);
+ }
- FileSystem fs = seqFileDir.getFileSystem(conf);
- for (FileStatus seqFile : fs.globStatus(new Path(seqFileDir, "part-*"))) {
- Path path = seqFile.getPath();
- System.out.println("Input Path: " + path);
- SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
- Writable key = (Writable) reader.getKeyClass().newInstance();
- Writable value = (Writable) reader.getValueClass().newInstance();
- while (reader.next(key, value)) {
- Cluster cluster = (Cluster) value;
- String fmtStr = useJSON ? cluster.asJsonString() : cluster.asFormatString(dictionary);
- if (subString > 0 && fmtStr.length() > subString) {
- writer.append(":").append(fmtStr.substring(0, Math.min(subString, fmtStr.length())));
- } else {
- writer.append(fmtStr);
- }
-
- writer.append('\n');
+ writer.write('\n');
- if (dictionary != null) {
- String topTerms = getTopFeatures(cluster.getCenter(), dictionary, numTopFeatures);
- writer.write("\tTop Terms: ");
- writer.write(topTerms);
- writer.write('\n');
- }
+ if (dictionary != null) {
+ String topTerms = getTopFeatures(cluster.getCenter(), dictionary, numTopFeatures);
+ writer.write("\tTop Terms: ");
+ writer.write(topTerms);
+ writer.write('\n');
+ }
- List<WeightedVectorWritable> points = clusterIdToPoints.get(cluster.getId());
- if (points != null) {
- writer.write("\tWeight: Point:\n\t");
- for (Iterator<WeightedVectorWritable> iterator = points.iterator(); iterator.hasNext();) {
- WeightedVectorWritable point = iterator.next();
- writer.append(Double.toString(point.getWeight())).append(": ");
- writer.append(ClusterBase.formatVector(point.getVector().get(), dictionary));
- if (iterator.hasNext()) {
- writer.append("\n\t");
+ List<WeightedVectorWritable> points = clusterIdToPoints.get(cluster.getId());
+ if (points != null) {
+ writer.write("\tWeight: Point:\n\t");
+ for (Iterator<WeightedVectorWritable> iterator = points.iterator(); iterator.hasNext();) {
+ WeightedVectorWritable point = iterator.next();
+ writer.write(String.valueOf(point.getWeight()));
+ writer.write(": ");
+ writer.write(ClusterBase.formatVector(point.getVector().get(), dictionary));
+ if (iterator.hasNext()) {
+ writer.write("\n\t");
+ }
+ }
+ writer.write('\n');
}
}
- writer.write('\n');
+ } finally {
+ reader.close();
}
- writer.flush();
}
- reader.close();
- }
- if (this.outputFile != null) {
- writer.flush();
+ } finally {
writer.close();
}
}
@@ -216,24 +219,27 @@ public final class ClusterDumper {
abuilder.withName("substring").withMinimum(1).withMaximum(1).create()).withDescription(
"The number of chars of the asFormatString() to print").withShortName("b").create();
Option numWordsOpt = obuilder.withLongName("numWords").withRequired(false).withArgument(
- abuilder.withName("numWords").withMinimum(1).withMaximum(1).create()).withDescription("The number of top terms to print")
- .withShortName("n").create();
+ abuilder.withName("numWords").withMinimum(1).withMaximum(1).create())
+ .withDescription("The number of top terms to print").withShortName("n").create();
Option centroidJSonOpt = obuilder.withLongName("json").withRequired(false).withDescription(
- "Output the centroid as JSON. Otherwise it substitues in the terms for vector cell entries").withShortName("j").create();
+ "Output the centroid as JSON. Otherwise it substitues in the terms for vector cell entries")
+ .withShortName("j").create();
Option pointsOpt = obuilder.withLongName("pointsDir").withRequired(false).withArgument(
abuilder.withName("pointsDir").withMinimum(1).withMaximum(1).create()).withDescription(
"The directory containing points sequence files mapping input vectors to their cluster. "
- + "If specified, then the program will output the points associated with a cluster").withShortName("p").create();
+ + "If specified, then the program will output the points associated with a cluster")
+ .withShortName("p").create();
Option dictOpt = obuilder.withLongName("dictionary").withRequired(false).withArgument(
- abuilder.withName("dictionary").withMinimum(1).withMaximum(1).create()).withDescription("The dictionary file. ")
- .withShortName("d").create();
+ abuilder.withName("dictionary").withMinimum(1).withMaximum(1).create())
+ .withDescription("The dictionary file. ").withShortName("d").create();
Option dictTypeOpt = obuilder.withLongName("dictionaryType").withRequired(false).withArgument(
abuilder.withName("dictionaryType").withMinimum(1).withMaximum(1).create()).withDescription(
"The dictionary file type (text|sequencefile)").withShortName("dt").create();
Option helpOpt = obuilder.withLongName("help").withDescription("Print out help").withShortName("h").create();
Group group = gbuilder.withName("Options").withOption(helpOpt).withOption(seqOpt).withOption(outputOpt)
- .withOption(substringOpt).withOption(pointsOpt).withOption(centroidJSonOpt).withOption(dictOpt).withOption(dictTypeOpt)
+ .withOption(substringOpt).withOption(pointsOpt).withOption(centroidJSonOpt)
+ .withOption(dictOpt).withOption(dictTypeOpt)
.withOption(numWordsOpt).create();
try {
@@ -348,7 +354,7 @@ public final class ClusterDumper {
static class TermIndexWeight {
private int index = -1;
- private double weight = 0.0;
+ private double weight;
TermIndexWeight(int index, double weight) {
this.index = index;
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/CollocMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/CollocMapper.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/CollocMapper.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/CollocMapper.java Wed Jun 2 12:37:36 2010
@@ -179,13 +179,11 @@ public class CollocMapper extends MapRed
return true;
}
});
- }
- catch (IllegalStateException ise) {
+ } catch (IllegalStateException ise) {
// catch an re-throw original exceptions from the procedures.
if (ise.getCause() instanceof IOException) {
throw (IOException) ise.getCause();
- }
- else {
+ } else {
// wasn't what was expected, so re-throw
throw ise;
}
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/Gram.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/Gram.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/Gram.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/Gram.java Wed Jun 2 12:37:36 2010
@@ -148,7 +148,6 @@ public class Gram extends BinaryComparab
/**
* @return gram frequency
- * @return
*/
public int getFrequency() {
return frequency;
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/GramKeyPartitioner.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/GramKeyPartitioner.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/GramKeyPartitioner.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/GramKeyPartitioner.java Wed Jun 2 12:37:36 2010
@@ -22,8 +22,9 @@ import org.apache.hadoop.io.WritableComp
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Partitioner;
-/** Partition GramKeys based on their Gram, ignoring the secondary sort key so that all GramKeys with the same
- * gram are sent to the same partition.
+/**
+ * Partition GramKeys based on their Gram, ignoring the secondary sort key so that all GramKeys with the same
+ * gram are sent to the same partition.
*/
public class GramKeyPartitioner implements Partitioner<GramKey, Gram> {
@@ -38,9 +39,9 @@ public class GramKeyPartitioner implemen
@Override
public int getPartition(GramKey key, Gram value, int numPartitions) {
// see: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/lib/partition/BinaryPartitioner.java?revision=816664&view=markup
- int length = key.getLength()-1;
- int right = (offset + length) % length;
- int hash = WritableComparator.hashBytes(key.getBytes(), right);
+ int length = key.getLength() - 1;
+ int right = (offset + length) % length;
+ int hash = WritableComparator.hashBytes(key.getBytes(), right);
return (hash & Integer.MAX_VALUE) % numPartitions;
}
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterable.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterable.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterable.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/SequenceFileVectorIterable.java Wed Jun 2 12:37:36 2010
@@ -34,7 +34,7 @@ import org.apache.mahout.math.VectorWrit
*/
public class SequenceFileVectorIterable implements Iterable<Vector> {
private final SequenceFile.Reader reader;
- private boolean transpose = false;
+ private boolean transpose;
public SequenceFileVectorIterable(SequenceFile.Reader reader) {
this.reader = reader;
@@ -82,15 +82,14 @@ public class SequenceFileVectorIterable
@Override
public Vector next() {
-
- return (transpose ? (VectorWritable)key : (VectorWritable)value).get();
+ return (transpose ? (VectorWritable) key : (VectorWritable) value).get();
}
/**
* Only valid when {@link #next()} is also valid
* @return The current Key
*/
- public Writable key(){
+ public Writable key() {
return transpose ? value : key;
}
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorDumper.java Wed Jun 2 12:37:36 2010
@@ -99,7 +99,7 @@ public final class VectorDumper {
if (cmdLine.hasOption(seqOpt)) {
Path path = new Path(cmdLine.getValue(seqOpt).toString());
- System.out.println("Input Path: " + path);
+ //System.out.println("Input Path: " + path); interferes with output?
JobClient client = new JobClient();
JobConf conf = new JobConf(Job.class);
client.setConf(conf);
@@ -113,9 +113,9 @@ public final class VectorDumper {
String[] dictionary = null;
if (cmdLine.hasOption(dictOpt)) {
- if (dictionaryType.equals("text")) {
+ if ("text".equals(dictionaryType)) {
dictionary = VectorHelper.loadTermDictionary(new File(cmdLine.getValue(dictOpt).toString()));
- } else if (dictionaryType.equals("sequencefile")) {
+ } else if ("sequencefile".equals(dictionaryType)) {
dictionary = VectorHelper.loadTermDictionary(conf, fs, cmdLine.getValue(dictOpt).toString());
} else {
throw new OptionException(dictTypeOpt);
@@ -124,33 +124,33 @@ public final class VectorDumper {
boolean useJSON = cmdLine.hasOption(centroidJSonOpt);
SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
- SequenceFileVectorIterable vectorIterable = new SequenceFileVectorIterable(reader, cmdLine
- .hasOption(vectorAsKeyOpt));
- Writer writer;
- if (cmdLine.hasOption(outputOpt)) {
- writer = new FileWriter(cmdLine.getValue(outputOpt).toString());
- } else {
- writer = new OutputStreamWriter(System.out);
- }
- boolean printKey = cmdLine.hasOption(printKeyOpt);
- SeqFileIterator iterator = (SeqFileIterator) vectorIterable.iterator();
- int i = 0;
- while (iterator.hasNext()) {
- Vector vector = iterator.next();
- if (printKey) {
- writer.write(iterator.key().toString());
- writer.write("\t");
+ try {
+ Iterable<Vector> vectorIterable = new SequenceFileVectorIterable(reader, cmdLine.hasOption(vectorAsKeyOpt));
+ Writer writer = cmdLine.hasOption(outputOpt)
+ ? new FileWriter(cmdLine.getValue(outputOpt).toString())
+ : new OutputStreamWriter(System.out);
+ try {
+ boolean printKey = cmdLine.hasOption(printKeyOpt);
+ SeqFileIterator iterator = (SeqFileIterator) vectorIterable.iterator();
+ //int i = 0;
+ while (iterator.hasNext()) {
+ Vector vector = iterator.next();
+ if (printKey) {
+ writer.write(iterator.key().toString());
+ writer.write("\t");
+ }
+ String fmtStr = useJSON ? vector.asFormatString() : VectorHelper.vectorToString(vector, dictionary);
+ writer.write(fmtStr);
+ writer.write('\n');
+ //i++;
+ }
+ //System.out.println("Dumped " + i + " Vectors");
+ } finally {
+ writer.close();
}
- String fmtStr = useJSON ? vector.asFormatString() : VectorHelper.vectorToString(vector, dictionary);
- writer.write(fmtStr);
- writer.write('\n');
- i++;
- }
- writer.flush();
- if (cmdLine.hasOption(outputOpt)) {
- writer.close();
+ } finally {
+ reader.close();
}
- System.out.println("Dumped " + i + " Vectors");
}
} catch (OptionException e) {
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorHelper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorHelper.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorHelper.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/VectorHelper.java Wed Jun 2 12:37:36 2010
@@ -34,9 +34,9 @@ import org.apache.hadoop.io.Text;
import org.apache.mahout.common.FileLineIterator;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.map.OpenObjectIntHashMap;
-import org.apache.mahout.utils.vectors.text.DictionaryVectorizer;
-public class VectorHelper {
+public final class VectorHelper {
+
private static final Pattern TAB_PATTERN = Pattern.compile("\t");
private VectorHelper() { }
@@ -76,7 +76,6 @@ public class VectorHelper {
* </pre>
*
* @param dictFile
- * @return
* @throws IOException
*/
public static String[] loadTermDictionary(File dictFile) throws IOException {
@@ -84,13 +83,13 @@ public class VectorHelper {
}
/**
- * Read a dictionary in {@link SequenceFile} generated by {@link DictionaryVectorizer}
+ * Read a dictionary in {@link SequenceFile} generated by
+ * {@link org.apache.mahout.utils.vectors.text.DictionaryVectorizer}
*
* @param conf
* @param fs
* @param filePattern
* <PATH TO DICTIONARY>/dictionary.file-*
- * @return
* @throws IOException
*/
public static String[] loadTermDictionary(Configuration conf, FileSystem fs, String filePattern) throws IOException {
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFModel.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFModel.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFModel.java Wed Jun 2 12:37:36 2010
@@ -28,7 +28,6 @@ import java.util.Map;
* ones may require a better implementation.
*/
public interface ARFFModel {
- DateFormat DEFAULT_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
String ARFF_SPARSE = "{"; //indicates the vector is sparse
String ARFF_COMMENT = "%";
String ATTRIBUTE = "@attribute";
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/ARFFVectorIterable.java Wed Jun 2 12:37:36 2010
@@ -29,6 +29,7 @@ import java.nio.charset.Charset;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Iterator;
+import java.util.Locale;
import java.util.NoSuchElementException;
import java.util.regex.Pattern;
@@ -85,7 +86,7 @@ public class ARFFVectorIterable implemen
//boolean inData = false; // TODO not used?
while ((line = buff.readLine()) != null) {
line = line.trim();
- String lower = line.toLowerCase();
+ String lower = line.toLowerCase(Locale.ENGLISH);
Integer labelNumInt = labelNumber;
if (lower.startsWith(ARFFModel.ARFF_COMMENT)) {
continue;
@@ -116,7 +117,7 @@ public class ARFFVectorIterable implemen
label = ARFFType.DATE.getLabel(lower);
type = ARFFType.DATE;
//TODO: DateFormatter map
- DateFormat format = ARFFModel.DEFAULT_DATE_FORMAT;
+ DateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
int idx = lower.indexOf(ARFFType.DATE.getIndicator());
String[] split = ARFFVectorIterable.SPACE_PATTERN.split(line);
if (split.length >= 4) { //we have a date format
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/Driver.java Wed Jun 2 12:37:36 2010
@@ -24,6 +24,7 @@ import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.nio.charset.Charset;
import java.util.Map;
@@ -41,6 +42,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.mahout.common.CommandLineUtil;
+import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
import org.apache.mahout.utils.vectors.io.JWriterVectorWriter;
import org.apache.mahout.utils.vectors.io.SequenceFileVectorWriter;
@@ -137,14 +139,19 @@ public final class Driver {
writeFile(outWriter, outDir, input, maxDocs, model);
}
log.info("Dictionary Output file: {}", dictOut);
- BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dictOut),
- Charset.forName("UTF8")));
Map<String,Integer> labels = model.getLabelBindings();
- for (Map.Entry<String,Integer> entry : labels.entrySet()) {
- writer.append(entry.getKey()).append(delimiter).append(String.valueOf(entry.getValue())).append(
- '\n');
+ Writer writer = new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(dictOut), Charset.forName("UTF8")));
+ try {
+ for (Map.Entry<String,Integer> entry : labels.entrySet()) {
+ writer.write(entry.getKey());
+ writer.write(delimiter);
+ writer.write(String.valueOf(entry.getValue()));
+ writer.write('\n');
+ }
+ } finally {
+ writer.close();
}
- writer.close();
}
} catch (OptionException e) {
@@ -158,24 +165,25 @@ public final class Driver {
log.info("Converting File: {}", file);
ARFFModel model = new MapBackedARFFModel(arffModel.getWords(), arffModel.getWordCount() + 1, arffModel
.getNominalMap());
- ARFFVectorIterable iteratable = new ARFFVectorIterable(file, model);
+ Iterable<Vector> iteratable = new ARFFVectorIterable(file, model);
String outFile = outDir + '/' + file.getName() + ".mvc";
VectorWriter vectorWriter;
- if (outWriter != null) {
- if (outWriter.equals("file")) {
- BufferedWriter writer = new BufferedWriter(new FileWriter(outFile));
- vectorWriter = new JWriterVectorWriter(writer);
+ if (outWriter == null) {
+ vectorWriter = getSeqFileWriter(outFile);
+ } else {
+ if ("file".equals(outWriter)) {
+ vectorWriter = new JWriterVectorWriter(new BufferedWriter(new FileWriter(outFile)));
} else {
vectorWriter = getSeqFileWriter(outFile);
}
- } else {
- vectorWriter = getSeqFileWriter(outFile);
}
-
- long numDocs = vectorWriter.write(iteratable, maxDocs);
- vectorWriter.close();
- log.info("Wrote: {} vectors", numDocs);
+ try {
+ long numDocs = vectorWriter.write(iteratable, maxDocs);
+ log.info("Wrote: {} vectors", numDocs);
+ } finally {
+ vectorWriter.close();
+ }
}
private static VectorWriter getSeqFileWriter(String outFile) throws IOException {
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/MapBackedARFFModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/MapBackedARFFModel.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/MapBackedARFFModel.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/arff/MapBackedARFFModel.java Wed Jun 2 12:37:36 2010
@@ -19,6 +19,7 @@ package org.apache.mahout.utils.vectors.
import java.text.DateFormat;
import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
@@ -123,13 +124,7 @@ public class MapBackedARFFModel implemen
return result;
}
-
- /**
- * Process a String
- *
- * @param data
- * @return
- */
+
// Not sure how scalable this is going to be
protected double processString(String data) {
data = MapBackedARFFModel.QUOTE_PATTERN.matcher(data).replaceAll("");
@@ -149,14 +144,11 @@ public class MapBackedARFFModel implemen
protected double processDate(String data, int idx) {
DateFormat format = dateMap.get(idx);
if (format == null) {
- format = DEFAULT_DATE_FORMAT;
+ format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
}
double result;
try {
- Date date;
- synchronized (format) {
- date = format.parse(data);
- }
+ Date date = format.parse(data);
result = date.getTime(); // hmmm, what kind of loss casting long to double?
} catch (ParseException e) {
throw new IllegalStateException(e);
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java Wed Jun 2 12:37:36 2010
@@ -47,11 +47,11 @@ public class CachedTermInfo implements T
int count = 0;
do {
Term term = te.term();
- if (term == null || term.field().equals(field) == false){
+ if (term == null || term.field().equals(field) == false) {
break;
}
int df = te.docFreq();
- if (df < minDf || df > percent){
+ if (df < minDf || df > percent) {
continue;
}
TermEntry entry = new TermEntry(term.text(), count++, df);
@@ -67,7 +67,9 @@ public class CachedTermInfo implements T
@Override
public TermEntry getTermEntry(String field, String term) {
- if (this.field.equals(field) == false){ return null; }
+ if (this.field.equals(field) == false) {
+ return null;
+ }
return termEntries.get(term);
}
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java Wed Jun 2 12:37:36 2010
@@ -144,31 +144,34 @@ public class ClusterLabels {
public void getLabels() throws IOException {
- Writer writer;
- if (this.output != null) {
- writer = new FileWriter(this.output);
- } else {
- writer = new OutputStreamWriter(System.out);
- }
-
- for (Map.Entry<Integer, List<WeightedVectorWritable>> integerListEntry : clusterIdToPoints.entrySet()) {
- List<WeightedVectorWritable> wvws = integerListEntry.getValue();
- List<TermInfoClusterInOut> termInfos = getClusterLabels(integerListEntry.getKey(), wvws);
- if (termInfos != null) {
- writer.write('\n');
- writer.write("Top labels for Cluster " + integerListEntry.getKey() + " containing " + wvws.size() + " vectors");
- writer.write('\n');
- writer.write("Term \t\t LLR \t\t In-ClusterDF \t\t Out-ClusterDF ");
- writer.write('\n');
- for (TermInfoClusterInOut termInfo : termInfos) {
- writer.write(termInfo.term + "\t\t" + termInfo.logLikelihoodRatio + "\t\t" + termInfo.inClusterDF + "\t\t"
- + termInfo.outClusterDF);
+ Writer writer = this.output == null ? new OutputStreamWriter(System.out) : new FileWriter(this.output);
+ try {
+ for (Map.Entry<Integer, List<WeightedVectorWritable>> integerListEntry : clusterIdToPoints.entrySet()) {
+ List<WeightedVectorWritable> wvws = integerListEntry.getValue();
+ List<TermInfoClusterInOut> termInfos = getClusterLabels(integerListEntry.getKey(), wvws);
+ if (termInfos != null) {
writer.write('\n');
+ writer.write("Top labels for Cluster ");
+ writer.write(String.valueOf(integerListEntry.getKey()));
+ writer.write(" containing ");
+ writer.write(String.valueOf(wvws.size()));
+ writer.write(" vectors");
+ writer.write('\n');
+ writer.write("Term \t\t LLR \t\t In-ClusterDF \t\t Out-ClusterDF ");
+ writer.write('\n');
+ for (TermInfoClusterInOut termInfo : termInfos) {
+ writer.write(termInfo.term);
+ writer.write("\t\t");
+ writer.write(String.valueOf(termInfo.logLikelihoodRatio));
+ writer.write("\t\t");
+ writer.write(String.valueOf(termInfo.inClusterDF));
+ writer.write("\t\t");
+ writer.write(String.valueOf(termInfo.outClusterDF));
+ writer.write('\n');
+ }
}
}
- }
- writer.flush();
- if (this.output != null) {
+ } finally {
writer.close();
}
}
@@ -273,7 +276,7 @@ public class ClusterLabels {
new SetBasedFieldSelector(Collections.singleton(idField), Collections.<String>emptySet());
for (int i = 0; i < numDocs; i++) {
- String id = null;
+ String id;
// Use Lucene's internal ID if idField is not specified. Else, get it from the document.
if (idField == null) {
id = Integer.toString(i);
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterable.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterable.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterable.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterable.java Wed Jun 2 12:37:36 2010
@@ -18,11 +18,8 @@
package org.apache.mahout.utils.vectors.lucene;
import java.io.IOException;
-import java.util.Collections;
import java.util.Iterator;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.SetBasedFieldSelector;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermDocs;
import org.apache.mahout.math.Vector;
@@ -87,7 +84,7 @@ public class LuceneIterable implements I
}
}
- private class TDIterator implements Iterator<Vector> {
+ private final class TDIterator implements Iterator<Vector> {
private final TermDocs termDocs;
private TDIterator() throws IOException {
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/text/term/TFPartialVectorReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/text/term/TFPartialVectorReducer.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/text/term/TFPartialVectorReducer.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/text/term/TFPartialVectorReducer.java Wed Jun 2 12:37:36 2010
@@ -76,11 +76,10 @@ public class TFPartialVectorReducer exte
do {
String term = ((TermAttribute) sf.getAttribute(TermAttribute.class)).term();
if (term.length() > 0) { // ngram
- if (dictionary.containsKey(term) == false) {
- continue;
+ if (dictionary.containsKey(term)) {
+ int termId = dictionary.get(term);
+ vector.setQuick(termId, vector.getQuick(termId) + 1);
}
- int termId = dictionary.get(term);
- vector.setQuick(termId, vector.getQuick(termId) + 1);
}
} while (sf.incrementToken());
@@ -89,11 +88,10 @@ public class TFPartialVectorReducer exte
} else {
for (String term : value.getEntries()) {
if (term.length() > 0) { // unigram
- if (dictionary.containsKey(term) == false) {
- continue;
+ if (dictionary.containsKey(term)) {
+ int termId = dictionary.get(term);
+ vector.setQuick(termId, vector.getQuick(termId) + 1);
}
- int termId = dictionary.get(term);
- vector.setQuick(termId, vector.getQuick(termId) + 1);
}
}
}
@@ -101,7 +99,7 @@ public class TFPartialVectorReducer exte
vector = new SequentialAccessSparseVector(vector);
}
// if the vector has no nonZero entries (nothing in the dictionary), let's not waste space sending it to disk.
- if(vector.getNumNondefaultElements() > 0) {
+ if (vector.getNumNondefaultElements() > 0) {
VectorWritable vectorWritable = new VectorWritable(vector);
output.collect(key, vectorWritable);
} else {
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/text/term/TermCountMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/text/term/TermCountMapper.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/text/term/TermCountMapper.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/text/term/TermCountMapper.java Wed Jun 2 12:37:36 2010
@@ -31,7 +31,6 @@ import org.apache.mahout.math.map.OpenOb
/**
* TextVectorizer Term Count Mapper. Tokenizes a text document and outputs the count of the words
- *
*/
public class TermCountMapper extends MapReduceBase implements Mapper<Text,StringTuple,Text,LongWritable> {
@Override
@@ -41,10 +40,10 @@ public class TermCountMapper extends Map
final Reporter reporter) throws IOException {
OpenObjectLongHashMap<String> wordCount = new OpenObjectLongHashMap<String>();
for (String word : value.getEntries()) {
- if (wordCount.containsKey(word) == false) {
- wordCount.put(word, 1);
- } else {
+ if (wordCount.containsKey(word)) {
wordCount.put(word, wordCount.get(word) + 1);
+ } else {
+ wordCount.put(word, 1);
}
}
wordCount.forEachPair(new ObjectLongProcedure<String>() {
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/tfidf/TFIDFPartialVectorReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/tfidf/TFIDFPartialVectorReducer.java?rev=950537&r1=950536&r2=950537&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/tfidf/TFIDFPartialVectorReducer.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/vectors/tfidf/TFIDFPartialVectorReducer.java Wed Jun 2 12:37:36 2010
@@ -53,7 +53,7 @@ public class TFIDFPartialVectorReducer e
private int minDf = 1;
private int maxDfPercent = 99;
private long vectorCount = 1;
- private long featureCount = 0;
+ private long featureCount;
private boolean sequentialAccess;
@Override