You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by ss...@apache.org on 2013/06/08 10:44:43 UTC
svn commit: r1490935 - in /mahout/trunk: ./
integration/src/main/java/org/apache/mahout/utils/vectors/lucene/
integration/src/test/java/org/apache/mahout/clustering/
integration/src/test/java/org/apache/mahout/utils/vectors/lucene/
Author: ssc
Date: Sat Jun 8 08:44:43 2013
New Revision: 1490935
URL: http://svn.apache.org/r1490935
Log:
MAHOUT-1243: Dictionary file format in Lucene-Mahout integration is not in SequenceFileFormat
Added:
mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/DriverTest.java
Modified:
mahout/trunk/CHANGELOG
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java
mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterEvaluator.java
Modified: mahout/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/mahout/trunk/CHANGELOG?rev=1490935&r1=1490934&r2=1490935&view=diff
==============================================================================
--- mahout/trunk/CHANGELOG (original)
+++ mahout/trunk/CHANGELOG Sat Jun 8 08:44:43 2013
@@ -2,15 +2,17 @@ Mahout Change Log
Release 0.8 - unreleased
+ MAHOUT-1243: Dictionary file format in Lucene-Mahout integration is not in SequenceFileFormat (ssc)
+
MAHOUT-974: org.apache.mahout.cf.taste.hadoop.als.ParallelALSFactorizationJob use integer as userId and itemId (ssc)
MAHOUT-1052: Add an option to MinHashDriver that specifies the dimension of vector to hash (indexes or values) (Elena Smirnova via smarthi)
-__MAHOUT-1237: Total cluster cost isn't computed properly (dfilimon)
+ MAHOUT-1237: Total cluster cost isn't computed properly (dfilimon)
MAHOUT-1196: LogisticModelParameters uses csv.getTargetCategories() even if csv is not used. (Vineet Krishnan via ssc)
-__MAHOUT-1224: Add the option of running a StreamingKMeans pass in the Reducer before BallKMeans (dfilimon)
+ MAHOUT-1224: Add the option of running a StreamingKMeans pass in the Reducer before BallKMeans (dfilimon)
MAHOUT-993: Some vector dumper flags are expecting arguments. (Andrew Look via robinanil)
@@ -28,21 +30,21 @@ __MAHOUT-1224: Add the option of running
MAHOUT-1213: SSVD job doesn't clean it's temp dir, and fails when seeing it again (smarthi)
-__MAHOUT-1223: Fixed point skipped in StreamingKMeans when iterating through centroids from a reducer (dfilimon)
+ MAHOUT-1223: Fixed point skipped in StreamingKMeans when iterating through centroids from a reducer (dfilimon)
-__MAHOUT-1222: Fix total weight in FastProjectionSearch (dfilimon)
+ MAHOUT-1222: Fix total weight in FastProjectionSearch (dfilimon)
-__MAHOUT-1219: Remove LSHSearcher from StreamingKMeansTest. It causes it to sometimes fail (dfilimon)
+ MAHOUT-1219: Remove LSHSearcher from StreamingKMeansTest. It causes it to sometimes fail (dfilimon)
MAHOUT-1221: SparseMatrix.viewRow is sometimes readonly. (Maysam Yabandeh via smarthi)
-__MAHOUT-1219: Remove LSHSearcher from SearchQualityTest. It causes it to fail, but the failure is not very meaningful (dfilimon)
+ MAHOUT-1219: Remove LSHSearcher from SearchQualityTest. It causes it to fail, but the failure is not very meaningful (dfilimon)
-__MAHOUT-1217: Nearest neighbor searchers sometimes fail to remove points: fix in FastProjectionSearch's searchFirst (dfilimon)
+ MAHOUT-1217: Nearest neighbor searchers sometimes fail to remove points: fix in FastProjectionSearch's searchFirst (dfilimon)
-__MAHOUT-1216: Add locality sensitive hashing and a LocalitySensitiveHash searcher (dfilimon)
+ MAHOUT-1216: Add locality sensitive hashing and a LocalitySensitiveHash searcher (dfilimon)
-__MAHOUT-1181: Adding StreamingKMeans MapReduce classes (dfilimon)
+ MAHOUT-1181: Adding StreamingKMeans MapReduce classes (dfilimon)
MAHOUT-1212: Incorrect classify-20newsgroups.sh file description (Julian Ortega via smarthi)
@@ -74,8 +76,7 @@ __MAHOUT-1181: Adding StreamingKMeans Ma
MAHOUT-1188: Inconsistent reference to Lucene versions in code and POM (smarthi)
- MAHOUT-1161: Unable to run CJKAnalyzer for conversion of a sequence file to sparse vector due to instantiation
- exception (ssc)
+ MAHOUT-1161: Unable to run CJKAnalyzer for conversion of a sequence file to sparse vector due to instantiation exception (ssc)
MAHOUT-1187: Update Commons Lang to Commons Lang3 (smarthi)
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java?rev=1490935&r1=1490934&r2=1490935&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java Sat Jun 8 08:44:43 2013
@@ -20,6 +20,7 @@ package org.apache.mahout.utils.vectors.
import java.io.File;
import java.io.IOException;
import java.io.Writer;
+import java.util.Iterator;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
@@ -36,14 +37,17 @@ import org.apache.commons.cli2.commandli
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.mahout.common.CommandLineUtil;
import org.apache.mahout.math.VectorWritable;
+import org.apache.mahout.utils.vectors.TermEntry;
import org.apache.mahout.utils.vectors.TermInfo;
import org.apache.mahout.utils.vectors.io.DelimitedTermInfoWriter;
import org.apache.mahout.utils.vectors.io.SequenceFileVectorWriter;
@@ -63,6 +67,7 @@ public final class Driver {
private String field;
private String idField;
private String dictOut;
+ private String seqDictOut = "";
private String weightType = "tfidf";
private String delimiter = "\t";
private double norm = LuceneIterable.NO_NORMALIZING;
@@ -98,10 +103,10 @@ public final class Driver {
LuceneIterable iterable;
if (norm == LuceneIterable.NO_NORMALIZING) {
- iterable = new LuceneIterable(reader, idField, field, termInfo,weight, LuceneIterable.NO_NORMALIZING,
+ iterable = new LuceneIterable(reader, idField, field, termInfo, weight, LuceneIterable.NO_NORMALIZING,
maxPercentErrorDocs);
} else {
- iterable = new LuceneIterable(reader, idField, field, termInfo,weight, norm, maxPercentErrorDocs);
+ iterable = new LuceneIterable(reader, idField, field, termInfo, weight, norm, maxPercentErrorDocs);
}
log.info("Output File: {}", outFile);
@@ -123,6 +128,31 @@ public final class Driver {
} finally {
Closeables.closeQuietly(tiWriter);
}
+
+ if (!"".equals(seqDictOut)) {
+ log.info("SequenceFile Dictionary Output file: {}", seqDictOut);
+
+ Path path = new Path(seqDictOut);
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.get(conf);
+ SequenceFile.Writer seqWriter = null;
+ try {
+ seqWriter = SequenceFile.createWriter(fs, conf, path, Text.class, IntWritable.class);
+ Text term = new Text();
+ IntWritable termIndex = new IntWritable();
+
+ Iterator<TermEntry> termEntries = termInfo.getAllEntries();
+ while (termEntries.hasNext()) {
+ TermEntry termEntry = termEntries.next();
+ term.set(termEntry.getTerm());
+ termIndex.set(termEntry.getTermIdx());
+ seqWriter.append(term, termIndex);
+ }
+ } finally {
+ Closeables.closeQuietly(seqWriter);
+ }
+
+ }
}
public static void main(String[] args) throws IOException {
@@ -152,6 +182,10 @@ public final class Driver {
abuilder.withName("dictOut").withMinimum(1).withMaximum(1).create()).withDescription(
"The output of the dictionary").withShortName("t").create();
+ Option seqDictOutOpt = obuilder.withLongName("seqDictOut").withRequired(false).withArgument(
+ abuilder.withName("seqDictOut").withMinimum(1).withMaximum(1).create()).withDescription(
+ "The output of the dictionary as sequence file").withShortName("st").create();
+
Option weightOpt = obuilder.withLongName("weight").withRequired(false).withArgument(
abuilder.withName("weight").withMinimum(1).withMaximum(1).create()).withDescription(
"The kind of weight to use. Currently TF or TFIDF").withShortName("w").create();
@@ -190,7 +224,7 @@ public final class Driver {
Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(idFieldOpt).withOption(
outputOpt).withOption(delimiterOpt).withOption(helpOpt).withOption(fieldOpt).withOption(maxOpt)
- .withOption(dictOutOpt).withOption(powerOpt).withOption(maxDFPercentOpt)
+ .withOption(dictOutOpt).withOption(seqDictOutOpt).withOption(powerOpt).withOption(maxDFPercentOpt)
.withOption(weightOpt).withOption(minDFOpt).withOption(maxPercentErrorDocsOpt).create();
try {
@@ -249,6 +283,10 @@ public final class Driver {
luceneDriver.setDictOut(cmdLine.getValue(dictOutOpt).toString());
+ if (cmdLine.hasOption(seqDictOutOpt)) {
+ luceneDriver.setSeqDictOut(cmdLine.getValue(seqDictOutOpt).toString());
+ }
+
luceneDriver.dumpVectors();
}
} catch (OptionException e) {
@@ -313,6 +351,10 @@ public final class Driver {
this.dictOut = dictOut;
}
+ public void setSeqDictOut(String seqDictOut) {
+ this.seqDictOut = seqDictOut;
+ }
+
public void setMaxPercentErrorDocs(double maxPercentErrorDocs) {
this.maxPercentErrorDocs = maxPercentErrorDocs;
}
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterEvaluator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterEvaluator.java?rev=1490935&r1=1490934&r2=1490935&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterEvaluator.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterEvaluator.java Sat Jun 8 08:44:43 2013
@@ -333,7 +333,7 @@ public final class TestClusterEvaluator
Configuration conf = new Configuration();
MeanShiftCanopyDriver.run(conf, testdata, output, measure, kernelProfile, 2.1, 1.0, 0.001, 10, false, true, true);
int numIterations = 10;
- Path clustersIn = new Path(output, "clusters-7-final");
+ Path clustersIn = new Path(output, "clusters-8-final");
RepresentativePointsDriver.run(conf, clustersIn, new Path(output, "clusteredPoints"), output, measure,
numIterations, true);
//printRepPoints(numIterations);
Added: mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/DriverTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/DriverTest.java?rev=1490935&view=auto
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/DriverTest.java (added)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/DriverTest.java Sat Jun 8 08:44:43 2013
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.utils.vectors.lucene;
+
+import com.google.common.collect.Sets;
+import com.google.common.io.Closeables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.lucene.util.Version;
+import org.apache.mahout.common.MahoutTestCase;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+
+public class DriverTest extends MahoutTestCase {
+
+ private File indexDir;
+ private File outputDir;
+ private Configuration conf;
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ indexDir = getTestTempDir("intermediate");
+ indexDir.delete();
+ outputDir = getTestTempDir("output");
+ outputDir.delete();
+
+ conf = new Configuration();
+ }
+
+ private Document asDocument(String line) {
+ Document doc = new Document();
+ doc.add(new TextFieldWithTermVectors("text", line));
+ return doc;
+ }
+
+ static class TextFieldWithTermVectors extends Field {
+
+ public static final FieldType TYPE = new FieldType();
+
+ static {
+ TYPE.setIndexed(true);
+ TYPE.setOmitNorms(true);
+ TYPE.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS);
+ TYPE.setStored(true);
+ TYPE.setTokenized(true);
+ TYPE.setStoreTermVectors(true);
+ TYPE.freeze();
+ }
+
+ public TextFieldWithTermVectors(String name, String value) {
+ super(name, value, TYPE);
+ }
+ }
+
+ @Test
+ public void sequenceFileDictionary() throws IOException {
+
+ Directory index = new SimpleFSDirectory(indexDir);
+ Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_43);
+ IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, analyzer);
+ final IndexWriter writer = new IndexWriter(index, config);
+
+ try {
+ writer.addDocument(asDocument("One Ring to rule them all"));
+ writer.addDocument(asDocument("One Ring to find them,"));
+ writer.addDocument(asDocument("One Ring to bring them all"));
+ writer.addDocument(asDocument("and in the darkness bind them"));
+
+ } finally {
+ writer.close(true);
+ }
+
+ File seqDict = new File(outputDir, "dict.seq");
+
+ Driver.main(new String[] {
+ "--dir", indexDir.getAbsolutePath(),
+ "--output", new File(outputDir, "out").getAbsolutePath(),
+ "--field", "text",
+ "--dictOut", new File(outputDir, "dict.txt").getAbsolutePath(),
+ "--seqDictOut", seqDict.getAbsolutePath(),
+ });
+
+ SequenceFile.Reader reader = null;
+ Set<String> indexTerms = Sets.newHashSet();
+ try {
+ reader = new SequenceFile.Reader(FileSystem.getLocal(conf), new Path(seqDict.getAbsolutePath()), conf);
+ Text term = new Text();
+ IntWritable termIndex = new IntWritable();
+
+ while (reader.next(term, termIndex)) {
+ indexTerms.add(term.toString());
+ }
+ } finally {
+ Closeables.closeQuietly(reader);
+ }
+
+ Set<String> expectedIndexTerms = Sets.newHashSet("all", "bind", "bring", "darkness", "find", "one", "ring", "rule");
+
+ // should contain the same terms as expected
+ assertEquals(expectedIndexTerms.size(), Sets.union(expectedIndexTerms, indexTerms).size());
+ }
+}