You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by gs...@apache.org on 2009/07/29 03:01:24 UTC
svn commit: r798741 - in
/lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils:
SequenceFileDumper.java clustering/ClusterDumper.java strings/
strings/StringUtil.java
Author: gsingers
Date: Wed Jul 29 01:01:24 2009
New Revision: 798741
URL: http://svn.apache.org/viewvc?rev=798741&view=rev
Log:
dump out points with clusters
Added:
lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/strings/
lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/strings/StringUtil.java
Modified:
lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java
lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java
Modified: lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java?rev=798741&r1=798740&r2=798741&view=diff
==============================================================================
--- lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java (original)
+++ lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/SequenceFileDumper.java Wed Jul 29 01:01:24 2009
@@ -33,6 +33,7 @@
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.jobcontrol.Job;
+import org.apache.mahout.utils.strings.StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,7 +45,6 @@
public class SequenceFileDumper {
private static final Logger log = LoggerFactory.getLogger(SequenceFileDumper.class);
- private static final String LINE_SEP = System.getProperty("line.separator");
private SequenceFileDumper() {
}
@@ -83,7 +83,6 @@
if (cmdLine.hasOption(seqOpt)) {
Path path = new Path(cmdLine.getValue(seqOpt).toString());
- System.out.println("Input Path: " + path);
JobClient client = new JobClient();
JobConf conf = new JobConf(Job.class);
client.setConf(conf);
@@ -96,6 +95,7 @@
} else {
writer = new OutputStreamWriter(System.out);
}
+ writer.append("Input Path: ").append(String.valueOf(path)).append(StringUtil.LINE_SEP);
int sub = Integer.MAX_VALUE;
if (cmdLine.hasOption(substringOpt)) {
@@ -105,21 +105,23 @@
long count = 0;
Writable key = (Writable) reader.getKeyClass().newInstance();
Writable value = (Writable) reader.getValueClass().newInstance();
+ writer.append("Key class: ").append(String.valueOf(reader.getKeyClass())).append(" Value Class: ").append(String.valueOf(value.getClass())).append(StringUtil.LINE_SEP);
+ writer.flush();
if (countOnly == false) {
while (reader.next(key, value)) {
writer.append("Key: ").append(String.valueOf(key));
String str = value.toString();
writer.append(": Value: ").append(str.length() > sub ? str.substring(0, sub) : str);
- writer.write(LINE_SEP);
+ writer.write(StringUtil.LINE_SEP);
writer.flush();
count++;
}
- System.out.println("Count: " + String.valueOf(count) + LINE_SEP);
+ writer.append("Count: ").append(String.valueOf(count)).append(StringUtil.LINE_SEP);
} else {
while (reader.next(key, value)) {
count++;
}
- System.out.println("Count: " + String.valueOf(count) + LINE_SEP);
+ writer.append("Count: ").append(String.valueOf(count)).append(StringUtil.LINE_SEP);
}
writer.flush();
if (cmdLine.hasOption(outputOpt)) {
Modified: lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java?rev=798741&r1=798740&r2=798741&view=diff
==============================================================================
--- lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java (original)
+++ lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/clustering/ClusterDumper.java Wed Jul 29 01:01:24 2009
@@ -25,22 +25,30 @@
import org.apache.commons.cli2.builder.DefaultOptionBuilder;
import org.apache.commons.cli2.builder.GroupBuilder;
import org.apache.commons.cli2.commandline.Parser;
-import org.apache.commons.cli2.util.HelpFormatter;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.jobcontrol.Job;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
import org.apache.mahout.clustering.ClusterBase;
+import org.apache.mahout.matrix.Vector;
+import org.apache.mahout.utils.CommandLineUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.Writer;
import java.io.FileWriter;
+import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
public final class ClusterDumper {
@@ -64,10 +72,13 @@
Option substringOpt = obuilder.withLongName("substring").withRequired(false).withArgument(
abuilder.withName("substring").withMinimum(1).withMaximum(1).create()).
withDescription("The number of chars of the asFormatString() to print").withShortName("b").create();
+ Option pointsOpt = obuilder.withLongName("points").withRequired(false).withArgument(
+ abuilder.withName("points").withMinimum(1).withMaximum(1).create()).
+ withDescription("The points sequence file mapping input vectors to their cluster. If specified, then the program will output the points associated with a cluster").withShortName("p").create();
Option helpOpt = obuilder.withLongName("help").
withDescription("Print out help").withShortName("h").create();
- Group group = gbuilder.withName("Options").withOption(seqOpt).withOption(outputOpt).withOption(substringOpt).create();
+ Group group = gbuilder.withName("Options").withOption(seqOpt).withOption(outputOpt).withOption(substringOpt).withOption(pointsOpt).create();
try {
Parser parser = new Parser();
@@ -76,7 +87,7 @@
if (cmdLine.hasOption(helpOpt)) {
- printHelp(group);
+ CommandLineUtil.printHelp(group);
return;
}
@@ -88,7 +99,13 @@
client.setConf(conf);
FileSystem fs = FileSystem.get(path.toUri(), conf);
SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
-
+ Map<String, List<String>> clusterIdToPoints = null;
+ if (cmdLine.hasOption(pointsOpt)) {
+ //read in the points
+ clusterIdToPoints = readPoints(cmdLine.getValue(pointsOpt).toString(), conf);
+ } else {
+ clusterIdToPoints = Collections.emptyMap();
+ }
Writer writer = null;
if (cmdLine.hasOption(outputOpt)){
writer = new FileWriter(cmdLine.getValue(outputOpt).toString());
@@ -102,13 +119,22 @@
Writable key = (Writable) reader.getKeyClass().newInstance();
ClusterBase value = (ClusterBase) reader.getValueClass().newInstance();
while (reader.next(key, value)){
- writer.write(value.getId());
- writer.write(":");
- writer.write("name:" + value.getCenter().getName());
- writer.write(":");
- String fmtStr = value.getCenter().asFormatString();
- writer.write(fmtStr.substring(0, Math.min(sub, fmtStr.length())));
- writer.write(LINE_SEP);
+ Vector center = value.getCenter();
+ String fmtStr = center.asFormatString();
+ writer.append(String.valueOf(value.getId())).append(":").append("name:")
+ .append(center.getName()).append(":").append(fmtStr.substring(0, Math.min(sub, fmtStr.length()))).append(LINE_SEP);
+ List<String> points = clusterIdToPoints.get(String.valueOf(value.getId()));
+ if (points != null){
+ writer.write("\tPoints: ");
+ for (Iterator<String> iterator = points.iterator(); iterator.hasNext();) {
+ String point = iterator.next();
+ writer.append(point);
+ if (iterator.hasNext()){
+ writer.append(", ");
+ }
+ }
+ writer.write(LINE_SEP);
+ }
writer.flush();
}
if (cmdLine.hasOption(outputOpt)){
@@ -119,14 +145,37 @@
} catch (OptionException e) {
log.error("Exception", e);
- printHelp(group);
+ CommandLineUtil.printHelp(group);
}
}
- private static void printHelp(Group group) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.setGroup(group);
- formatter.print();
+ private static Map<String, List<String>> readPoints(String pointsPath, JobConf conf) throws IOException {
+ Map<String, List<String>> result = new HashMap<String, List<String>>();
+ Path path = new Path(pointsPath);
+ FileSystem fs = FileSystem.get(path.toUri(), conf);
+ SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
+ try {
+ Text key = (Text) reader.getKeyClass().newInstance();
+ Text value = (Text) reader.getValueClass().newInstance();
+ while (reader.next(key, value)) {
+ //value is the cluster id as an int, key is the name/id of the vector, but that doesn't matter because we only care about printing it
+ String clusterId = value.toString();
+ List<String> pointList = result.get(clusterId);
+ if (pointList == null) {
+ pointList = new ArrayList<String>();
+ result.put(clusterId, pointList);
+ }
+ pointList.add(key.toString());
+
+ }
+ } catch (InstantiationException e) {
+ log.error("Exception", e);
+ } catch (IllegalAccessException e) {
+ log.error("Exception", e);
+ }
+ return result;
}
+
+
}
\ No newline at end of file
Added: lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/strings/StringUtil.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/strings/StringUtil.java?rev=798741&view=auto
==============================================================================
--- lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/strings/StringUtil.java (added)
+++ lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/strings/StringUtil.java Wed Jul 29 01:01:24 2009
@@ -0,0 +1,11 @@
+package org.apache.mahout.utils.strings;
+
+
+/**
+ *
+ *
+ **/
+public class StringUtil {
+
+ public static final String LINE_SEP = System.getProperty("line.separator");
+}