You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by gs...@apache.org on 2013/01/30 11:27:18 UTC
svn commit: r1440334 [1/2] - in /mahout/trunk: ./ core/
core/src/main/java/org/apache/mahout/common/
core/src/main/java/org/apache/mahout/common/commandline/
core/src/main/java/org/apache/mahout/common/lucene/
core/src/main/java/org/apache/mahout/vecto...
Author: gsingers
Date: Wed Jan 30 10:27:17 2013
New Revision: 1440334
URL: http://svn.apache.org/viewvc?rev=1440334&view=rev
Log:
MAHOUT-1112: upgrade to Lucene 4.1.0
Added:
mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/AnalyzerUtils.java
mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfoTest.java
Modified:
mahout/trunk/core/pom.xml
mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java
mahout/trunk/core/src/main/java/org/apache/mahout/common/commandline/DefaultOptionCreator.java
mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/TokenStreamIterator.java
mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DefaultAnalyzer.java
mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/EncodingMapper.java
mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java
mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/TFIDF.java
mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java
mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapper.java
mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/document/SequenceFileTokenizerMapper.java
mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/encoders/LuceneTextValueEncoder.java
mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DictionaryVectorizerTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapperTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/encoders/TextValueEncoderTest.java
mahout/trunk/examples/pom.xml
mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/NewsgroupHelper.java
mahout/trunk/integration/pom.xml
mahout/trunk/integration/src/main/java/org/apache/mahout/text/MailArchivesClusteringAnalyzer.java
mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaAnalyzer.java
mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaDatasetCreatorMapper.java
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/regex/AnalyzerTransformer.java
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/VectorHelper.java
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterable.java
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterator.java
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/TFDFMapper.java
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/VectorMapper.java
mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java
mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/dirichlet/TestL1ModelClustering.java
mahout/trunk/integration/src/test/java/org/apache/mahout/text/MailArchivesClusteringAnalyzerTest.java
mahout/trunk/integration/src/test/java/org/apache/mahout/utils/nlp/collocations/llr/BloomTokenFilterTest.java
mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/LuceneIterableTest.java
mahout/trunk/pom.xml
Modified: mahout/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/mahout/trunk/core/pom.xml?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/pom.xml (original)
+++ mahout/trunk/core/pom.xml Wed Jan 30 10:27:17 2013
@@ -174,7 +174,7 @@
<dependency>
<groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers</artifactId>
+ <artifactId>lucene-analyzers-common</artifactId>
</dependency>
<dependency>
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java Wed Jan 30 10:27:17 2013
@@ -52,12 +52,14 @@ import org.apache.hadoop.mapreduce.lib.o
import org.apache.hadoop.util.Tool;
import org.apache.lucene.analysis.Analyzer;
import org.apache.mahout.common.commandline.DefaultOptionCreator;
+import org.apache.mahout.common.lucene.AnalyzerUtils;
import org.apache.mahout.math.VectorWritable;
-import org.apache.mahout.vectorizer.DefaultAnalyzer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.util.Version;
/**
* <p>Superclass of many Mahout Hadoop "jobs". A job drives configuration and launch of one or
@@ -589,13 +591,15 @@ public abstract class AbstractJob extend
}
protected Class<? extends Analyzer> getAnalyzerClassFromOption() throws ClassNotFoundException {
- Class<? extends Analyzer> analyzerClass = DefaultAnalyzer.class;
+ Class<? extends Analyzer> analyzerClass = StandardAnalyzer.class;
if (hasOption(DefaultOptionCreator.ANALYZER_NAME_OPTION)) {
String className = getOption(DefaultOptionCreator.ANALYZER_NAME_OPTION);
analyzerClass = Class.forName(className).asSubclass(Analyzer.class);
// try instantiating it, b/c there isn't any point in setting it if
// you can't instantiate it
- ClassUtils.instantiateAs(analyzerClass, Analyzer.class);
+ //ClassUtils.instantiateAs(analyzerClass, Analyzer.class);
+ AnalyzerUtils.createAnalyzer(analyzerClass);
+
}
return analyzerClass;
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/common/commandline/DefaultOptionCreator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/commandline/DefaultOptionCreator.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/commandline/DefaultOptionCreator.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/commandline/DefaultOptionCreator.java Wed Jan 30 10:27:17 2013
@@ -20,10 +20,11 @@ package org.apache.mahout.common.command
import org.apache.commons.cli2.Option;
import org.apache.commons.cli2.builder.ArgumentBuilder;
import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.mahout.clustering.meanshift.MeanShiftCanopyDriver;
import org.apache.mahout.common.distance.SquaredEuclideanDistanceMeasure;
import org.apache.mahout.common.kernel.TriangularKernelProfile;
-import org.apache.mahout.vectorizer.DefaultAnalyzer;
+
public final class DefaultOptionCreator {
@@ -338,7 +339,7 @@ public static DefaultOptionBuilder clust
.withDescription(
"If present, the name of a Lucene analyzer class to use")
.withArgument(
- new ArgumentBuilder().withName(ANALYZER_NAME_OPTION).withDefault(DefaultAnalyzer.class.getName())
+ new ArgumentBuilder().withName(ANALYZER_NAME_OPTION).withDefault(StandardAnalyzer.class.getName())
.withMinimum(1).withMaximum(1).create()
)
.withShortName("an");
Added: mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/AnalyzerUtils.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/AnalyzerUtils.java?rev=1440334&view=auto
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/AnalyzerUtils.java (added)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/AnalyzerUtils.java Wed Jan 30 10:27:17 2013
@@ -0,0 +1,74 @@
+package org.apache.mahout.common.lucene;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.util.Version;
+import org.apache.mahout.common.ClassUtils;
+
+/**
+ *
+ *
+ **/
+public class AnalyzerUtils {
+
+ /**
+ * Create an Analyzer using the latest {@link org.apache.lucene.util.Version}. Note, if you need to pass in parameters
+ * to your constructor, you will need to wrap it in an implementation that does not take any arguments
+ * @param analyzerClassName
+ * @return
+ * @throws ClassNotFoundException
+ */
+ public static Analyzer createAnalyzer(String analyzerClassName) throws ClassNotFoundException {
+ return createAnalyzer(analyzerClassName, Version.LUCENE_41);
+ }
+
+ public static Analyzer createAnalyzer(String analyzerClassName, Version version) throws ClassNotFoundException {
+ Analyzer analyzer = null;
+ Class<? extends Analyzer> analyzerClass = Class.forName(analyzerClassName).asSubclass(Analyzer.class);
+ //TODO: GSI: Not sure I like this, many analyzers in Lucene take in the version
+
+ return createAnalyzer(analyzerClass, version);
+ }
+
+ /**
+ * Create an Analyzer using the latest {@link org.apache.lucene.util.Version}. Note, if you need to pass in parameters
+ * to your constructor, you will need to wrap it in an implementation that does not take any arguments
+ * @param analyzerClass The Analyzer Class to instantiate
+ * @return
+ */
+ public static Analyzer createAnalyzer(Class<? extends Analyzer> analyzerClass){
+ return createAnalyzer(analyzerClass, Version.LUCENE_41);
+ }
+
+ public static Analyzer createAnalyzer(Class<? extends Analyzer> analyzerClass, Version version){
+ Analyzer analyzer = null;
+ if (analyzerClass == StandardAnalyzer.class) {
+ Class<?>[] params = new Class<?>[1];
+ params[0] = Version.class;
+ Object[] args = new Object[1];
+ args[0] = version;
+ analyzer = ClassUtils.instantiateAs(analyzerClass,
+ Analyzer.class, params, args);
+
+ } else {
+ analyzer = ClassUtils.instantiateAs(analyzerClass, Analyzer.class);
+ }
+ return analyzer;
+ }
+}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/TokenStreamIterator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/TokenStreamIterator.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/TokenStreamIterator.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/TokenStreamIterator.java Wed Jan 30 10:27:17 2013
@@ -25,6 +25,9 @@ import java.io.IOException;
/**
* Provide an Iterator for the tokens in a TokenStream.
+ *
+ * Note, it is the responsibility of the instantiating class to properly consume the {@link org.apache.lucene.analysis.TokenStream}. See
+ * the Lucene {@link org.apache.lucene.analysis.TokenStream} documentation for more information.
*/
//TODO: consider using the char/byte arrays instead of strings, esp. when we upgrade to Lucene 4.0
public final class TokenStreamIterator extends AbstractIterator<String> {
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DefaultAnalyzer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DefaultAnalyzer.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DefaultAnalyzer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DefaultAnalyzer.java Wed Jan 30 10:27:17 2013
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.mahout.vectorizer;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.util.Version;
-
-import java.io.IOException;
-import java.io.Reader;
-
-/**
- * A subclass of the Lucene StandardAnalyzer that provides a no-argument constructor.
- * Used as the default analyzer in many cases where an analyzer is instantiated by
- * class name by calling a no-arg constructor.
- */
-public final class DefaultAnalyzer extends Analyzer {
-
- private final StandardAnalyzer stdAnalyzer = new StandardAnalyzer(Version.LUCENE_36);
-
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return stdAnalyzer.tokenStream(fieldName, reader);
- }
-
- @Override
- public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
- return stdAnalyzer.reusableTokenStream(fieldName, reader);
- }
-}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/EncodingMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/EncodingMapper.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/EncodingMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/EncodingMapper.java Wed Jan 30 10:27:17 2013
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.mahout.vectorizer;
import org.apache.hadoop.conf.Configuration;
@@ -22,7 +21,9 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.util.Version;
import org.apache.mahout.common.ClassUtils;
+import org.apache.mahout.common.lucene.AnalyzerUtils;
import org.apache.mahout.math.NamedVector;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.SequentialAccessSparseVector;
@@ -44,7 +45,6 @@ public class EncodingMapper extends Mapp
public static final String ENCODER_FIELD_NAME = "encoderFieldName";
public static final String ENCODER_CLASS = "encoderClass";
public static final String CARDINALITY = "cardinality";
-
private boolean sequentialVecs;
private boolean namedVectors;
private FeatureVectorEncoder encoder;
@@ -56,14 +56,21 @@ public class EncodingMapper extends Mapp
sequentialVecs = conf.getBoolean(USE_SEQUENTIAL, false);
namedVectors = conf.getBoolean(USE_NAMED_VECTORS, false);
String analyzerName = conf.get(ANALYZER_NAME, StandardAnalyzer.class.getName());
- Analyzer analyzer = ClassUtils.instantiateAs(analyzerName, Analyzer.class);
+ Analyzer analyzer = null;
+ try {
+ analyzer = AnalyzerUtils.createAnalyzer(analyzerName);
+ } catch (ClassNotFoundException e) {
+ //TODO: hmmm, don't like this approach
+ throw new IOException("Unable to create Analyzer for name: " + analyzerName, e);
+ }
+
String encoderName = conf.get(ENCODER_FIELD_NAME, "text");
cardinality = conf.getInt(CARDINALITY, 5000);
String encClass = conf.get(ENCODER_CLASS);
encoder = ClassUtils.instantiateAs(encClass,
- FeatureVectorEncoder.class,
- new Class[]{String.class},
- new Object[]{encoderName});
+ FeatureVectorEncoder.class,
+ new Class[]{String.class},
+ new Object[]{encoderName});
if (encoder instanceof LuceneTextValueEncoder) {
((LuceneTextValueEncoder) encoder).setAnalyzer(analyzer);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java Wed Jan 30 10:27:17 2013
@@ -31,12 +31,14 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.ToolRunner;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.mahout.common.AbstractJob;
import org.apache.mahout.common.ClassUtils;
import org.apache.mahout.common.CommandLineUtil;
import org.apache.mahout.common.HadoopUtil;
import org.apache.mahout.common.Pair;
import org.apache.mahout.common.commandline.DefaultOptionCreator;
+import org.apache.mahout.common.lucene.AnalyzerUtils;
import org.apache.mahout.math.hadoop.stats.BasicStats;
import org.apache.mahout.vectorizer.collocations.llr.LLRReducer;
import org.apache.mahout.vectorizer.common.PartialVectorMerger;
@@ -196,13 +198,13 @@ public final class SparseVectorsFromSequ
}
log.info("Number of reduce tasks: {}", reduceTasks);
- Class<? extends Analyzer> analyzerClass = DefaultAnalyzer.class;
+ Class<? extends Analyzer> analyzerClass = StandardAnalyzer.class;
if (cmdLine.hasOption(analyzerNameOpt)) {
String className = cmdLine.getValue(analyzerNameOpt).toString();
analyzerClass = Class.forName(className).asSubclass(Analyzer.class);
// try instantiating it, b/c there isn't any point in setting it if
// you can't instantiate it
- ClassUtils.instantiateAs(analyzerClass, Analyzer.class);
+ AnalyzerUtils.createAnalyzer(analyzerClass);
}
boolean processIdf;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/TFIDF.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/TFIDF.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/TFIDF.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/TFIDF.java Wed Jan 30 10:27:17 2013
@@ -17,22 +17,24 @@
package org.apache.mahout.vectorizer;
-import org.apache.lucene.search.DefaultSimilarity;
-import org.apache.lucene.search.Similarity;
-
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+//TODO: add a new class that supports arbitrary Lucene similarity implementations
public class TFIDF implements Weight {
+
+ private DefaultSimilarity sim = new DefaultSimilarity();
+
+ public TFIDF() {
+ }
- private Similarity sim = new DefaultSimilarity();
-
- public TFIDF() { }
-
- public TFIDF(Similarity sim) {
+/* public TFIDF(Similarity sim) {
this.sim = sim;
}
-
+ */
+
+
@Override
public double calculate(int tf, int df, int length, int numDocs) {
- // ignore length
+ // ignore length
return sim.tf(tf) * sim.idf(df, numDocs);
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocDriver.java Wed Jan 30 10:27:17 2013
@@ -33,11 +33,12 @@ import org.apache.hadoop.mapreduce.lib.o
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.mahout.common.AbstractJob;
import org.apache.mahout.common.ClassUtils;
import org.apache.mahout.common.HadoopUtil;
import org.apache.mahout.common.commandline.DefaultOptionCreator;
-import org.apache.mahout.vectorizer.DefaultAnalyzer;
+import org.apache.mahout.common.lucene.AnalyzerUtils;
import org.apache.mahout.vectorizer.DocumentProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -130,14 +131,13 @@ public final class CollocDriver extends
if (argMap.containsKey("preprocess")) {
log.info("Input will be preprocessed");
-
- Class<? extends Analyzer> analyzerClass = DefaultAnalyzer.class;
+ Class<? extends Analyzer> analyzerClass = StandardAnalyzer.class;
if (getOption("analyzerName") != null) {
String className = getOption("analyzerName");
analyzerClass = Class.forName(className).asSubclass(Analyzer.class);
// try instantiating it, b/c there isn't any point in setting it if
// you can't instantiate it
- ClassUtils.instantiateAs(analyzerClass, Analyzer.class);
+ AnalyzerUtils.createAnalyzer(analyzerClass);
}
Path tokenizedPath = new Path(output, DocumentProcessor.TOKENIZED_DOCUMENT_OUTPUT_FOLDER);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapper.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapper.java Wed Jan 30 10:27:17 2013
@@ -91,7 +91,7 @@ public class CollocMapper extends Mapper
protected void map(Text key, StringTuple value, final Context context) throws IOException, InterruptedException {
ShingleFilter sf = new ShingleFilter(new IteratorTokenStream(value.getEntries().iterator()), maxShingleSize);
-
+ sf.reset();
try {
int count = 0; // ngram count
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/document/SequenceFileTokenizerMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/document/SequenceFileTokenizerMapper.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/document/SequenceFileTokenizerMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/document/SequenceFileTokenizerMapper.java Wed Jan 30 10:27:17 2013
@@ -20,15 +20,18 @@ package org.apache.mahout.vectorizer.doc
import java.io.IOException;
import java.io.StringReader;
+import com.google.common.io.Closeables;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.Version;
import org.apache.mahout.common.ClassUtils;
import org.apache.mahout.common.StringTuple;
-import org.apache.mahout.vectorizer.DefaultAnalyzer;
+import org.apache.mahout.common.lucene.AnalyzerUtils;
import org.apache.mahout.vectorizer.DocumentProcessor;
/**
@@ -40,7 +43,8 @@ public class SequenceFileTokenizerMapper
@Override
protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
- TokenStream stream = analyzer.reusableTokenStream(key.toString(), new StringReader(value.toString()));
+ TokenStream stream = analyzer.tokenStream(key.toString(), new StringReader(value.toString()));
+ stream.reset();
CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
StringTuple document = new StringTuple();
stream.reset();
@@ -49,14 +53,21 @@ public class SequenceFileTokenizerMapper
document.add(new String(termAtt.buffer(), 0, termAtt.length()));
}
}
+ stream.end();
+ Closeables.closeQuietly(stream);
context.write(key, document);
}
@Override
protected void setup(Context context) throws IOException, InterruptedException {
super.setup(context);
- analyzer = ClassUtils.instantiateAs(context.getConfiguration().get(DocumentProcessor.ANALYZER_CLASS,
- DefaultAnalyzer.class.getName()),
- Analyzer.class);
+
+ String analyzerClassName = context.getConfiguration().get(DocumentProcessor.ANALYZER_CLASS,
+ StandardAnalyzer.class.getName());
+ try {
+ analyzer = AnalyzerUtils.createAnalyzer(analyzerClassName);
+ } catch (ClassNotFoundException e) {
+ throw new IOException("Unable to create analyzer: " + analyzerClassName, e);
+ }
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/encoders/LuceneTextValueEncoder.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/encoders/LuceneTextValueEncoder.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/encoders/LuceneTextValueEncoder.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/encoders/LuceneTextValueEncoder.java Wed Jan 30 10:27:17 2013
@@ -22,7 +22,6 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.mahout.common.lucene.TokenStreamIterator;
-
import java.io.IOException;
import java.io.Reader;
import java.nio.CharBuffer;
@@ -30,6 +29,7 @@ import java.util.Iterator;
/**
* Encodes text using a lucene style tokenizer.
+ *
* @see TextValueEncoder
*/
public class LuceneTextValueEncoder extends TextValueEncoder {
@@ -50,9 +50,9 @@ public class LuceneTextValueEncoder exte
@Override
protected Iterable<String> tokenize(CharSequence originalForm) {
try {
- TokenStream ts = analyzer.reusableTokenStream(getName(), new CharSequenceReader(originalForm));
+ TokenStream ts = analyzer.tokenStream(getName(), new CharSequenceReader(originalForm));
ts.addAttribute(CharTermAttribute.class);
- return new LuceneTokenIterable(ts);
+ return new LuceneTokenIterable(ts, false);
} catch (IOException ex) {
throw new IllegalStateException(ex);
}
@@ -95,11 +95,11 @@ public class LuceneTextValueEncoder exte
}
@Override
- public void close() {
+ public void close() {
// do nothing
}
}
-
+ //GSI: TODO: we really need a way to make sure we call the TokenStream workflow here (i.e. end and close when we are done)
private static final class LuceneTokenIterable implements Iterable<String> {
private boolean firstTime = true;
private final TokenStream tokenStream;
@@ -108,6 +108,11 @@ public class LuceneTextValueEncoder exte
this.tokenStream = ts;
}
+ private LuceneTokenIterable(TokenStream ts, boolean firstTime) {
+ this.tokenStream = ts;
+ this.firstTime = firstTime;
+ }
+
/**
* Returns an iterator over a set of elements of type T.
*
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java Wed Jan 30 10:27:17 2013
@@ -73,6 +73,7 @@ public class TFPartialVectorReducer exte
if (maxNGramSize >= 2) {
ShingleFilter sf = new ShingleFilter(new IteratorTokenStream(value.getEntries().iterator()), maxNGramSize);
+ sf.reset();
try {
do {
String term = sf.getAttribute(CharTermAttribute.class).toString();
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DictionaryVectorizerTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DictionaryVectorizerTest.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DictionaryVectorizerTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DictionaryVectorizerTest.java Wed Jan 30 10:27:17 2013
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.Pair;
import org.apache.mahout.common.iterator.sequencefile.PathFilters;
@@ -95,7 +96,7 @@ public final class DictionaryVectorizerT
private void runTest(boolean sequential, boolean named)
throws IOException, ClassNotFoundException, InterruptedException {
- Class<? extends Analyzer> analyzer = DefaultAnalyzer.class;
+ Class<? extends Analyzer> analyzer = StandardAnalyzer.class;
Path tokenizedDocuments = getTestTempDirPath("output/tokenized-documents");
Path wordCount = getTestTempDirPath("output/wordcount");
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java Wed Jan 30 10:27:17 2013
@@ -31,6 +31,7 @@ import org.apache.mahout.common.iterator
import org.junit.Test;
import java.util.Arrays;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
/**
* Tests tokenizing of {@link SequenceFile}s containing document ID and text (both as {@link Text})
@@ -59,7 +60,7 @@ public class DocumentProcessorTest exten
Closeables.closeQuietly(writer);
}
- DocumentProcessor.tokenizeDocuments(input, DefaultAnalyzer.class, output, configuration);
+ DocumentProcessor.tokenizeDocuments(input, StandardAnalyzer.class, output, configuration);
FileStatus[] statuses = fs.listStatus(output, PathFilters.logsCRCFilter());
assertEquals(1, statuses.length);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapperTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapperTest.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapperTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapperTest.java Wed Jan 30 10:27:17 2013
@@ -186,16 +186,16 @@ public final class CollocMapperTest exte
}
/** A lucene 2.9 standard analyzer with no stopwords. */
- public static class TestAnalyzer extends Analyzer {
+ /* public static class TestAnalyzer extends DefaultAnalyzer {
private final Analyzer a;
public TestAnalyzer() {
- a = new StandardAnalyzer(Version.LUCENE_31, Collections.emptySet());
+ a = new StandardAnalyzer(Version.LUCENE_40, Collections.emptySet());
}
@Override
public TokenStream tokenStream(String arg0, Reader arg1) {
return a.tokenStream(arg0, arg1);
}
- }
+ }*/
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/encoders/TextValueEncoderTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/encoders/TextValueEncoderTest.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/encoders/TextValueEncoderTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/encoders/TextValueEncoderTest.java Wed Jan 30 10:27:17 2013
@@ -18,7 +18,7 @@
package org.apache.mahout.vectorizer.encoders;
import com.google.common.collect.ImmutableMap;
-import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.util.Version;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.math.DenseVector;
@@ -70,7 +70,7 @@ public final class TextValueEncoderTest
@Test
public void testLuceneEncoding() throws Exception {
LuceneTextValueEncoder enc = new LuceneTextValueEncoder("text");
- enc.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_34));
+ enc.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_41));
Vector v1 = new DenseVector(200);
enc.addToVector("test1 and more", v1);
enc.flush(1, v1);
Modified: mahout/trunk/examples/pom.xml
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/pom.xml?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/examples/pom.xml (original)
+++ mahout/trunk/examples/pom.xml Wed Jan 30 10:27:17 2013
@@ -141,7 +141,7 @@
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers</artifactId>
+ <artifactId>lucene-analyzers-common</artifactId>
</dependency>
<dependency>
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/NewsgroupHelper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/NewsgroupHelper.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/NewsgroupHelper.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/NewsgroupHelper.java Wed Jan 30 10:27:17 2013
@@ -46,7 +46,6 @@ import java.util.Locale;
import java.util.Random;
public final class NewsgroupHelper {
- private static final Version LUCENE_VERSION = Version.LUCENE_36;
private static final SimpleDateFormat[] DATE_FORMATS = {
new SimpleDateFormat("", Locale.ENGLISH),
@@ -60,7 +59,7 @@ public final class NewsgroupHelper {
private static final long WEEK = 7 * 24 * 3600;
private final Random rand = RandomUtils.getRandom();
- private final Analyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);
+ private final Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_41);
private final FeatureVectorEncoder encoder = new StaticWordValueEncoder("body");
private final FeatureVectorEncoder bias = new ConstantValueEncoder("Intercept");
@@ -118,7 +117,7 @@ public final class NewsgroupHelper {
Collection<String> words,
Reader in,
Multiset<String> overallCounts) throws IOException {
- TokenStream ts = analyzer.reusableTokenStream("text", in);
+ TokenStream ts = analyzer.tokenStream("text", in);
ts.addAttribute(CharTermAttribute.class);
ts.reset();
while (ts.incrementToken()) {
@@ -126,5 +125,7 @@ public final class NewsgroupHelper {
words.add(s);
}
overallCounts.addAll(words);
+ ts.end();
+ Closeables.closeQuietly(ts);
}
}
Modified: mahout/trunk/integration/pom.xml
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/pom.xml?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/pom.xml (original)
+++ mahout/trunk/integration/pom.xml Wed Jan 30 10:27:17 2013
@@ -142,7 +142,7 @@
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
- <artifactId>lucene-analyzers</artifactId>
+ <artifactId>lucene-analyzers-common</artifactId>
</dependency>
<dependency>
@@ -177,6 +177,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -189,6 +193,11 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+
</exclusions>
</dependency>
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/MailArchivesClusteringAnalyzer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/MailArchivesClusteringAnalyzer.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/MailArchivesClusteringAnalyzer.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/MailArchivesClusteringAnalyzer.java Wed Jan 30 10:27:17 2013
@@ -22,19 +22,19 @@ import java.util.Arrays;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.lucene.analysis.ASCIIFoldingFilter;
-import org.apache.lucene.analysis.CharArraySet;
-import org.apache.lucene.analysis.LowerCaseFilter;
-import org.apache.lucene.analysis.PorterStemFilter;
-import org.apache.lucene.analysis.StopFilter;
-import org.apache.lucene.analysis.StopwordAnalyzerBase;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.en.PorterStemFilter;
+import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
/**
@@ -43,12 +43,13 @@ import org.apache.lucene.util.Version;
* stop words, excluding non-alpha-numeric tokens, and porter stemming.
*/
public final class MailArchivesClusteringAnalyzer extends StopwordAnalyzerBase {
- private static final Version LUCENE_VERSION = Version.LUCENE_36;
+ private static final Version LUCENE_VERSION = Version.LUCENE_41;
// extended set of stop words composed of common mail terms like "hi",
// HTML tags, and Java keywords asmany of the messages in the archives
// are subversion check-in notifications
- private static final Set<?> STOP_WORDS = new CharArraySet(LUCENE_VERSION, Arrays.asList(
+
+private static CharArraySet stopSet = new CharArraySet(LUCENE_VERSION, Arrays.asList(
"3d","7bit","a0","about","above","abstract","across","additional","after",
"afterwards","again","against","align","all","almost","alone","along",
"already","also","although","always","am","among","amongst","amoungst",
@@ -108,11 +109,12 @@ public final class MailArchivesClusterin
private static final Matcher matcher = alphaNumeric.matcher("");
public MailArchivesClusteringAnalyzer() {
- super(LUCENE_VERSION, STOP_WORDS);
+ super(LUCENE_VERSION, stopSet);
}
- public MailArchivesClusteringAnalyzer(Set<?> stopSet) {
+ public MailArchivesClusteringAnalyzer(CharArraySet stopSet) {
super(LUCENE_VERSION, stopSet);
+
}
@Override
@@ -122,7 +124,7 @@ public final class MailArchivesClusterin
result = new LowerCaseFilter(LUCENE_VERSION, result);
result = new ASCIIFoldingFilter(result);
result = new AlphaNumericMaxLengthFilter(result);
- result = new StopFilter(LUCENE_VERSION, result, stopwords);
+ result = new StopFilter(LUCENE_VERSION, result, stopSet);
result = new PorterStemFilter(result);
return new TokenStreamComponents(tokenizer, result);
}
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaAnalyzer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaAnalyzer.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaAnalyzer.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaAnalyzer.java Wed Jan 30 10:27:17 2013
@@ -18,15 +18,15 @@
package org.apache.mahout.text.wikipedia;
import java.io.Reader;
-import java.util.Set;
-
-import org.apache.lucene.analysis.LowerCaseFilter;
-import org.apache.lucene.analysis.StopAnalyzer;
-import org.apache.lucene.analysis.StopFilter;
-import org.apache.lucene.analysis.StopwordAnalyzerBase;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.core.StopAnalyzer;
+import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer;
import org.apache.lucene.util.Version;
@@ -34,19 +34,19 @@ import org.apache.lucene.util.Version;
public class WikipediaAnalyzer extends StopwordAnalyzerBase {
public WikipediaAnalyzer() {
- super(Version.LUCENE_31, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+ super(Version.LUCENE_41, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
}
- public WikipediaAnalyzer(Set<?> stopSet) {
- super(Version.LUCENE_31, stopSet);
+ public WikipediaAnalyzer(CharArraySet stopSet) {
+ super(Version.LUCENE_41, stopSet);
}
-
+
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new WikipediaTokenizer(reader);
- TokenStream result = new StandardFilter(Version.LUCENE_31, tokenizer);
- result = new LowerCaseFilter(Version.LUCENE_31, result);
- result = new StopFilter(Version.LUCENE_31, result, stopwords);
+ TokenStream result = new StandardFilter(Version.LUCENE_41, tokenizer);
+ result = new LowerCaseFilter(Version.LUCENE_41, result);
+ result = new StopFilter(Version.LUCENE_41, result, super.getStopwordSet());
return new TokenStreamComponents(tokenizer, result);
}
-}
+}
\ No newline at end of file
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaDatasetCreatorMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaDatasetCreatorMapper.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaDatasetCreatorMapper.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaDatasetCreatorMapper.java Wed Jan 30 10:27:17 2013
@@ -25,6 +25,7 @@ import java.util.Locale;
import java.util.Set;
import java.util.regex.Pattern;
+import com.google.common.io.Closeables;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DefaultStringifier;
@@ -67,7 +68,7 @@ public class WikipediaDatasetCreatorMapp
String catMatch = findMatchingCategory(document);
if (!"Unknown".equals(catMatch)) {
StringBuilder contents = new StringBuilder(1000);
- TokenStream stream = analyzer.reusableTokenStream(catMatch, new StringReader(document));
+ TokenStream stream = analyzer.tokenStream(catMatch, new StringReader(document));
CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
stream.reset();
while (stream.incrementToken()) {
@@ -76,6 +77,8 @@ public class WikipediaDatasetCreatorMapp
context.write(
new Text(SPACE_NON_ALPHA_PATTERN.matcher(catMatch).replaceAll("_")),
new Text(contents.toString()));
+ stream.end();
+ Closeables.closeQuietly(stream);
}
}
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/regex/AnalyzerTransformer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/regex/AnalyzerTransformer.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/regex/AnalyzerTransformer.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/regex/AnalyzerTransformer.java Wed Jan 30 10:27:17 2013
@@ -17,6 +17,7 @@
package org.apache.mahout.utils.regex;
+import com.google.common.io.Closeables;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
@@ -33,7 +34,7 @@ public class AnalyzerTransformer impleme
private String fieldName = "text";
public AnalyzerTransformer() {
- this(new StandardAnalyzer(Version.LUCENE_34), "text");
+ this(new StandardAnalyzer(Version.LUCENE_41), "text");
}
public AnalyzerTransformer(Analyzer analyzer) {
@@ -48,15 +49,20 @@ public class AnalyzerTransformer impleme
@Override
public String transformMatch(String match) {
StringBuilder result = new StringBuilder();
+ TokenStream ts = null;
try {
- TokenStream ts = analyzer.reusableTokenStream(fieldName, new StringReader(match));
+ ts = analyzer.tokenStream(fieldName, new StringReader(match));
ts.addAttribute(CharTermAttribute.class);
+ ts.reset();
TokenStreamIterator iter = new TokenStreamIterator(ts);
while (iter.hasNext()) {
result.append(iter.next()).append(' ');
}
+ ts.end();
} catch (IOException e) {
throw new IllegalStateException(e);
+ } finally {
+ Closeables.closeQuietly(ts);
}
return result.toString();
}
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/VectorHelper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/VectorHelper.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/VectorHelper.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/VectorHelper.java Wed Jan 30 10:27:17 2013
@@ -49,8 +49,9 @@ public final class VectorHelper {
private static final Pattern TAB_PATTERN = Pattern.compile("\t");
-
- private VectorHelper() { }
+
+ private VectorHelper() {
+ }
public static String vectorToCSVString(Vector vector, boolean namesAsComments) throws IOException {
Appendable bldr = new StringBuilder(2048);
@@ -58,13 +59,13 @@ public final class VectorHelper {
return bldr.toString();
}
- public static String buildJson(Iterable<Pair<String,Double>> iterable) {
+ public static String buildJson(Iterable<Pair<String, Double>> iterable) {
return buildJson(iterable, new StringBuilder(2048));
}
- public static String buildJson(Iterable<Pair<String,Double>> iterable, StringBuilder bldr) {
+ public static String buildJson(Iterable<Pair<String, Double>> iterable, StringBuilder bldr) {
bldr.append('{');
- for (Pair<String,Double> p : iterable) {
+ for (Pair<String, Double> p : iterable) {
bldr.append(p.getFirst());
bldr.append(':');
bldr.append(p.getSecond());
@@ -81,7 +82,7 @@ public final class VectorHelper {
}
public static List<Pair<Integer, Double>> topEntries(Vector vector, int maxEntries) {
- PriorityQueue<Pair<Integer,Double>> queue = new TDoublePQ<Integer>(-1, maxEntries);
+ PriorityQueue<Pair<Integer, Double>> queue = new TDoublePQ<Integer>(-1, maxEntries);
Iterator<Vector.Element> it = vector.iterateNonZero();
while (it.hasNext()) {
Vector.Element e = it.next();
@@ -110,30 +111,30 @@ public final class VectorHelper {
}
public static List<Pair<String, Double>> toWeightedTerms(Collection<Pair<Integer, Double>> entries,
- final String[] dictionary) {
+ final String[] dictionary) {
if (dictionary != null) {
return Lists.newArrayList(Collections2.transform(entries,
- new Function<Pair<Integer, Double>, Pair<String, Double>>() {
- @Override
- public Pair<String, Double> apply(Pair<Integer, Double> p) {
- return Pair.of(dictionary[p.getFirst()], p.getSecond());
- }
- }));
+ new Function<Pair<Integer, Double>, Pair<String, Double>>() {
+ @Override
+ public Pair<String, Double> apply(Pair<Integer, Double> p) {
+ return Pair.of(dictionary[p.getFirst()], p.getSecond());
+ }
+ }));
} else {
return Lists.newArrayList(Collections2.transform(entries,
- new Function<Pair<Integer, Double>, Pair<String, Double>>() {
- @Override
- public Pair<String, Double> apply(Pair<Integer, Double> p) {
- return Pair.of(Integer.toString(p.getFirst()), p.getSecond());
- }
- }));
+ new Function<Pair<Integer, Double>, Pair<String, Double>>() {
+ @Override
+ public Pair<String, Double> apply(Pair<Integer, Double> p) {
+ return Pair.of(Integer.toString(p.getFirst()), p.getSecond());
+ }
+ }));
}
}
public static String vectorToJson(Vector vector, String[] dictionary, int maxEntries, boolean sort) {
return buildJson(toWeightedTerms(sort
- ? topEntries(vector, maxEntries)
- : firstEntries(vector, maxEntries), dictionary));
+ ? topEntries(vector, maxEntries)
+ : firstEntries(vector, maxEntries), dictionary));
}
public static void vectorToCSVString(Vector vector,
@@ -155,10 +156,10 @@ public final class VectorHelper {
}
bldr.append('\n');
}
-
+
/**
* Read in a dictionary file. Format is:
- *
+ * <p/>
* <pre>
* term DocFreq Index
* </pre>
@@ -166,19 +167,18 @@ public final class VectorHelper {
public static String[] loadTermDictionary(File dictFile) throws IOException {
return loadTermDictionary(new FileInputStream(dictFile));
}
-
+
/**
* Read a dictionary in {@link SequenceFile} generated by
* {@link org.apache.mahout.vectorizer.DictionaryVectorizer}
*
- * @param filePattern
- * <PATH TO DICTIONARY>/dictionary.file-*
+ * @param filePattern <PATH TO DICTIONARY>/dictionary.file-*
*/
public static String[] loadTermDictionary(Configuration conf, String filePattern) {
OpenObjectIntHashMap<String> dict = new OpenObjectIntHashMap<String>();
- for (Pair<Text,IntWritable> record :
- new SequenceFileDirIterable<Text,IntWritable>(new Path(filePattern), PathType.GLOB,
- null, null, true, conf)) {
+ for (Pair<Text, IntWritable> record :
+ new SequenceFileDirIterable<Text, IntWritable>(new Path(filePattern), PathType.GLOB,
+ null, null, true, conf)) {
dict.put(record.getFirst().toString(), record.getSecond().get());
}
String[] dictionary = new String[dict.size()];
@@ -187,20 +187,20 @@ public final class VectorHelper {
}
return dictionary;
}
-
+
/**
* Read in a dictionary file. Format is: First line is the number of entries
- *
+ * <p/>
* <pre>
* term DocFreq Index
* </pre>
*/
private static String[] loadTermDictionary(InputStream is) throws IOException {
FileLineIterator it = new FileLineIterator(is);
-
+
int numEntries = Integer.parseInt(it.next());
String[] result = new String[numEntries];
-
+
while (it.hasNext()) {
String line = it.next();
if (line.startsWith("#")) {
@@ -218,15 +218,18 @@ public final class VectorHelper {
private static class TDoublePQ<T> extends PriorityQueue<Pair<T, Double>> {
private final T sentinel;
+
private TDoublePQ(T sentinel, int size) {
- initialize(size);
+ super(size);
this.sentinel = sentinel;
}
+
@Override
protected boolean lessThan(Pair<T, Double> a,
- Pair<T, Double> b) {
+ Pair<T, Double> b) {
return a.getSecond().compareTo(b.getSecond()) < 0;
}
+
@Override
protected Pair<T, Double> getSentinelObject() {
return Pair.of(sentinel, Double.NEGATIVE_INFINITY);
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfo.java Wed Jan 30 10:27:17 2013
@@ -17,58 +17,54 @@
package org.apache.mahout.utils.vectors.lucene;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.BytesRef;
+import org.apache.mahout.utils.vectors.TermEntry;
+import org.apache.mahout.utils.vectors.TermInfo;
+
import java.io.IOException;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
-import com.google.common.io.Closeables;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
-import org.apache.mahout.utils.vectors.TermEntry;
-import org.apache.mahout.utils.vectors.TermInfo;
-
/**
* Caches TermEntries from a single field. Materializes all values in the TermEnum to memory (much like FieldCache)
*/
public class CachedTermInfo implements TermInfo {
-
+
private final Map<String, TermEntry> termEntries;
private final String field;
-
+
public CachedTermInfo(IndexReader reader, String field, int minDf, int maxDfPercent) throws IOException {
this.field = field;
- TermEnum te = reader.terms(new Term(field, ""));
- try {
- int numDocs = reader.numDocs();
- double percent = numDocs * maxDfPercent / 100.0;
- //Should we use a linked hash map so that we know terms are in order?
- termEntries = new LinkedHashMap<String, TermEntry>();
- int count = 0;
- do {
- Term term = te.term();
- if (term == null || !term.field().equals(field)) {
- break;
- }
- int df = te.docFreq();
- if (df < minDf || df > percent) {
- continue;
- }
- TermEntry entry = new TermEntry(term.text(), count++, df);
- termEntries.put(entry.getTerm(), entry);
- } while (te.next());
- } finally {
- Closeables.closeQuietly(te);
+ Terms t = MultiFields.getTerms(reader, field);
+ TermsEnum te = t.iterator(null);
+
+ int numDocs = reader.numDocs();
+ double percent = numDocs * maxDfPercent / 100.0;
+ //Should we use a linked hash map so that we know terms are in order?
+ termEntries = new LinkedHashMap<String, TermEntry>();
+ int count = 0;
+ BytesRef text;
+ while ((text = te.next()) != null) {
+ int df = te.docFreq();
+ if (df < minDf || df > percent) {
+ continue;
+ }
+ TermEntry entry = new TermEntry(text.utf8ToString(), count++, df);
+ termEntries.put(entry.getTerm(), entry);
}
}
-
+
@Override
public int totalTerms(String field) {
return termEntries.size();
}
-
+
@Override
public TermEntry getTermEntry(String field, String term) {
if (!this.field.equals(field)) {
@@ -76,7 +72,7 @@ public class CachedTermInfo implements T
}
return termEntries.get(term);
}
-
+
@Override
public Iterator<TermEntry> getAllEntries() {
return termEntries.values().iterator();
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/ClusterLabels.java Wed Jan 30 10:27:17 2013
@@ -29,9 +29,12 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.util.BytesRef;
import com.google.common.base.Charsets;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
+import java.util.Set;
+import java.util.TreeSet;
import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
import org.apache.commons.cli2.Option;
@@ -41,14 +44,17 @@ import org.apache.commons.cli2.builder.D
import org.apache.commons.cli2.builder.GroupBuilder;
import org.apache.commons.cli2.commandline.Parser;
import org.apache.hadoop.fs.Path;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.SetBasedFieldSelector;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.OpenBitSet;
import org.apache.mahout.clustering.classify.WeightedVectorWritable;
import org.apache.mahout.common.CommandLineUtil;
@@ -95,8 +101,6 @@ public class ClusterLabels {
this.contentField = contentField;
this.minNumIds = minNumIds;
this.maxLabels = maxLabels;
- this.minNumIds = DEFAULT_MIN_IDS;
- this.maxLabels = DEFAULT_MAX_LABELS;
ClusterDumper clusterDumper = new ClusterDumper(seqFileDir, pointsDir);
this.clusterIdToPoints = clusterDumper.getClusterIdToPoints();
}
@@ -153,8 +157,9 @@ public class ClusterLabels {
log.info("Processing Cluster {} with {} documents", integer, wvws.size());
Directory dir = FSDirectory.open(new File(this.indexDir));
- IndexReader reader = IndexReader.open(dir, false);
-
+ IndexReader reader = DirectoryReader.open(dir);
+
+
log.info("# of documents in the index {}", reader.numDocs());
Collection<String> idSet = new HashSet<String>();
@@ -180,37 +185,32 @@ public class ClusterLabels {
* frequencies in each document. The number of results of this call will be the in-cluster document
* frequency.
*/
-
- TermEnum te = reader.terms(new Term(contentField, ""));
+ Terms t = MultiFields.getTerms(reader, contentField);
+ TermsEnum te = t.iterator(null);
Map<String, TermEntry> termEntryMap = new LinkedHashMap<String, TermEntry>();
+ Bits liveDocs = MultiFields.getLiveDocs(reader); //WARNING: returns null if there are no deletions
- try {
- int count = 0;
- do {
- Term term = te.term();
- if (term == null || !term.field().equals(contentField)) {
- break;
+ int count = 0;
+ BytesRef term;
+ while ((term = te.next()) != null) {
+ OpenBitSet termBitset = new OpenBitSet(reader.maxDoc());
+ DocsEnum docsEnum = MultiFields.getTermDocsEnum(reader, null, contentField, term);
+ int docID = 0;
+ while ((docID = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+ if (liveDocs != null && !liveDocs.get(docID)) { //check to see if we don't have an deletions (null) or if document is live
+ // document is deleted...
+ termBitset.set(docsEnum.docID());
}
- OpenBitSet termBitset = new OpenBitSet(reader.maxDoc());
-
- // Generate bitset for the term
- TermDocs termDocs = reader.termDocs(term);
+ }
+ // AND the term's bitset with cluster doc bitset to get the term's in-cluster frequency.
+ // This modifies the termBitset, but that's fine as we are not using it anywhere else.
+ termBitset.and(clusterDocBitset);
+ int inclusterDF = (int) termBitset.cardinality();
- while (termDocs.next()) {
- termBitset.set(termDocs.doc());
- }
+ TermEntry entry = new TermEntry(term.utf8ToString(), count++, inclusterDF);
+ termEntryMap.put(entry.getTerm(), entry);
- // AND the term's bitset with cluster doc bitset to get the term's in-cluster frequency.
- // This modifies the termBitset, but that's fine as we are not using it anywhere else.
- termBitset.and(clusterDocBitset);
- int inclusterDF = (int) termBitset.cardinality();
-
- TermEntry entry = new TermEntry(term.text(), count++, inclusterDF);
- termEntryMap.put(entry.getTerm(), entry);
- } while (te.next());
- } finally {
- Closeables.closeQuietly(te);
}
List<TermInfoClusterInOut> clusteredTermInfo = new LinkedList<TermInfoClusterInOut>();
@@ -218,7 +218,8 @@ public class ClusterLabels {
int clusterSize = wvws.size();
for (TermEntry termEntry : termEntryMap.values()) {
- int corpusDF = reader.terms(new Term(this.contentField, termEntry.getTerm())).docFreq();
+
+ int corpusDF = reader.docFreq(new Term(this.contentField,termEntry.getTerm()));
int outDF = corpusDF - termEntry.getDocFreq();
int inDF = termEntry.getDocFreq();
double logLikelihoodRatio = scoreDocumentFrequencies(inDF, outDF, clusterSize, numDocs);
@@ -241,10 +242,14 @@ public class ClusterLabels {
int numDocs = reader.numDocs();
OpenBitSet bitset = new OpenBitSet(numDocs);
-
- FieldSelector idFieldSelector =
- new SetBasedFieldSelector(Collections.singleton(idField), Collections.<String>emptySet());
-
+
+ Set<String> idFieldSelector= null;
+ if(idField !=null){
+ idFieldSelector= new TreeSet<String>();
+ idFieldSelector.add(idField);
+ }
+
+
for (int i = 0; i < numDocs; i++) {
String id;
// Use Lucene's internal ID if idField is not specified. Else, get it from the document.
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/Driver.java Wed Jan 30 10:27:17 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -81,7 +82,8 @@ public final class Driver {
Preconditions.checkArgument(maxDFPercent <= 99, "maxDFPercent must be <= 99");
Directory dir = FSDirectory.open(file);
- IndexReader reader = IndexReader.open(dir, true);
+ IndexReader reader = DirectoryReader.open(dir);
+
Weight weight;
if ("tf".equalsIgnoreCase(weightType)) {
@@ -93,13 +95,12 @@ public final class Driver {
}
TermInfo termInfo = new CachedTermInfo(reader, field, minDf, maxDFPercent);
- VectorMapper mapper = new TFDFMapper(reader, weight, termInfo);
-
+
LuceneIterable iterable;
if (norm == LuceneIterable.NO_NORMALIZING) {
- iterable = new LuceneIterable(reader, idField, field, mapper, LuceneIterable.NO_NORMALIZING, maxPercentErrorDocs);
+ iterable = new LuceneIterable(reader, idField, field, termInfo,weight, LuceneIterable.NO_NORMALIZING, maxPercentErrorDocs);
} else {
- iterable = new LuceneIterable(reader, idField, field, mapper, norm, maxPercentErrorDocs);
+ iterable = new LuceneIterable(reader, idField, field, termInfo,weight, norm, maxPercentErrorDocs);
}
log.info("Output File: {}", outFile);
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterable.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterable.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterable.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterable.java Wed Jan 30 10:27:17 2013
@@ -17,11 +17,13 @@
package org.apache.mahout.utils.vectors.lucene;
-import java.io.IOException;
-import java.util.Iterator;
-
import org.apache.lucene.index.IndexReader;
import org.apache.mahout.math.Vector;
+import org.apache.mahout.utils.vectors.TermInfo;
+import org.apache.mahout.vectorizer.Weight;
+
+import java.io.IOException;
+import java.util.Iterator;
/**
* {@link Iterable} counterpart to {@link LuceneIterator}.
@@ -33,46 +35,48 @@ public final class LuceneIterable implem
private final IndexReader indexReader;
private final String field;
private final String idField;
- private final VectorMapper mapper;
+ private final TermInfo terminfo;
private final double normPower;
private final double maxPercentErrorDocs;
+ private final Weight weight;
- public LuceneIterable(IndexReader reader, String idField, String field, VectorMapper mapper) {
- this(reader, idField, field, mapper, NO_NORMALIZING);
+ public LuceneIterable(IndexReader reader, String idField, String field, TermInfo terminfo, Weight weight) {
+ this(reader, idField, field, terminfo, weight, NO_NORMALIZING);
}
-
- public LuceneIterable(IndexReader indexReader, String idField, String field, VectorMapper mapper, double normPower) {
- this(indexReader, idField, field, mapper, normPower, 0);
+
+ public LuceneIterable(IndexReader indexReader, String idField, String field, TermInfo terminfo, Weight weight, double normPower) {
+ this(indexReader, idField, field, terminfo, weight, normPower, 0);
}
-
+
/**
* Produce a LuceneIterable that can create the Vector plus normalize it.
- *
- * @param indexReader {@link org.apache.lucene.index.IndexReader} to read the documents from.
- * @param idField field containing the id. May be null.
- * @param field field to use for the Vector
- * @param mapper {@link VectorMapper} for creating {@link Vector}s from Lucene's TermVectors.
- * @param normPower the normalization value. Must be nonnegative, or {@link #NO_NORMALIZING}
+ *
+ * @param indexReader {@link org.apache.lucene.index.IndexReader} to read the documents from.
+ * @param idField field containing the id. May be null.
+ * @param field field to use for the Vector
+ * @param normPower the normalization value. Must be nonnegative, or {@link #NO_NORMALIZING}
* @param maxPercentErrorDocs the percentage of documents in the lucene index that can have a null term vector
*/
public LuceneIterable(IndexReader indexReader,
String idField,
String field,
- VectorMapper mapper,
+ TermInfo terminfo,
+ Weight weight,
double normPower,
double maxPercentErrorDocs) {
this.indexReader = indexReader;
this.idField = idField;
this.field = field;
- this.mapper = mapper;
+ this.terminfo = terminfo;
this.normPower = normPower;
this.maxPercentErrorDocs = maxPercentErrorDocs;
+ this.weight = weight;
}
-
+
@Override
public Iterator<Vector> iterator() {
try {
- return new LuceneIterator(indexReader, idField, field, mapper, normPower, maxPercentErrorDocs);
+ return new LuceneIterator(indexReader, idField, field, terminfo, weight, normPower, maxPercentErrorDocs);
} catch (IOException e) {
throw new IllegalStateException(e);
}
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterator.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterator.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/LuceneIterator.java Wed Jan 30 10:27:17 2013
@@ -17,23 +17,26 @@
package org.apache.mahout.utils.vectors.lucene;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-
import com.google.common.base.Preconditions;
import com.google.common.collect.AbstractIterator;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.SetBasedFieldSelector;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermFreqVector;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.BytesRef;
import org.apache.mahout.math.NamedVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.utils.Bump125;
+import org.apache.mahout.utils.vectors.TermInfo;
+import org.apache.mahout.vectorizer.Weight;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.TreeSet;
+
/**
* An {@link Iterator} over {@link Vector}s that uses a Lucene index as the source for creating the
* {@link Vector}s. The field used to create the vectors currently must have term vectors stored for it.
@@ -45,56 +48,73 @@ public final class LuceneIterator extend
private final IndexReader indexReader;
private final String field;
private final String idField;
- private final FieldSelector idFieldSelector;
- private final VectorMapper mapper;
+ private final Set<String> idFieldSelector;
+ private final TermInfo terminfo;
private final double normPower;
- private final TermDocs termDocs;
+ private DocsEnum termDocs = null;
+
+
+ private int nextDocid;
+
private int numErrorDocs = 0;
private int maxErrorDocs = 0;
private final Bump125 bump = new Bump125();
private long nextLogRecord = bump.increment();
private int skippedErrorMessages = 0;
+ private Weight weight;
/**
* Produce a LuceneIterable that can create the Vector plus normalize it.
*
* @param indexReader {@link IndexReader} to read the documents from.
- * @param idField field containing the id. May be null.
- * @param field field to use for the Vector
- * @param mapper {@link VectorMapper} for creating {@link Vector}s from Lucene's TermVectors.
- * @param normPower the normalization value. Must be nonnegative, or {@link LuceneIterable#NO_NORMALIZING}
+ * @param idField field containing the id. May be null.
+ * @param field field to use for the Vector
+ * @param normPower the normalization value. Must be nonnegative, or {@link LuceneIterable#NO_NORMALIZING}
*/
public LuceneIterator(IndexReader indexReader,
String idField,
String field,
- VectorMapper mapper,
+ TermInfo terminfo,
+ Weight weight,
double normPower) throws IOException {
- this(indexReader, idField, field, mapper, normPower, 0.0);
+ this(indexReader, idField, field, terminfo, weight, normPower, 0.0);
}
/**
- * @see #LuceneIterator(IndexReader, String, String, VectorMapper, double)
* @param maxPercentErrorDocs most documents that will be tolerated without a term freq vector. In [0,1].
+ * @see #LuceneIterator(org.apache.lucene.index.IndexReader, String, String, org.apache.mahout.utils.vectors.TermInfo, org.apache.mahout.vectorizer.Weight, double)
*/
public LuceneIterator(IndexReader indexReader,
String idField,
String field,
- VectorMapper mapper,
+ TermInfo terminfo,
+ Weight weight,
double normPower,
double maxPercentErrorDocs) throws IOException {
// term docs(null) is a better way of iterating all the docs in Lucene
Preconditions.checkArgument(normPower == LuceneIterable.NO_NORMALIZING || normPower >= 0,
- "If specified normPower must be nonnegative", normPower);
+ "If specified normPower must be nonnegative", normPower);
Preconditions.checkArgument(maxPercentErrorDocs >= 0.0 && maxPercentErrorDocs <= 1.0);
- idFieldSelector = new SetBasedFieldSelector(Collections.singleton(idField), Collections.<String>emptySet());
+ if (idField != null) {
+ idFieldSelector = new TreeSet();
+ idFieldSelector.add(idField);
+ } else {
+ idFieldSelector = null; /*The field in the index containing the index. If
+ null, then the Lucene
+ internal doc id is used
+ which is prone to error
+ if the underlying index
+ changes*/
+
+ }
this.indexReader = indexReader;
this.idField = idField;
this.field = field;
- this.mapper = mapper;
+ this.terminfo = terminfo;
this.normPower = normPower;
- // term docs(null) is a better way of iterating all the docs in Lucene
- this.termDocs = indexReader.termDocs(null);
+ this.weight = weight;
+ this.nextDocid = 0;
this.maxErrorDocs = (int) (maxPercentErrorDocs * indexReader.numDocs());
}
@@ -102,15 +122,17 @@ public final class LuceneIterator extend
protected Vector computeNext() {
try {
int doc;
- TermFreqVector termFreqVector;
+ Terms termFreqVector;
do {
- if (!termDocs.next()) {
+ doc = this.nextDocid;
+ nextDocid++;
+
+ if (doc >= indexReader.maxDoc()) {
return endOfData();
}
- doc = termDocs.doc();
- termFreqVector = indexReader.getTermFreqVector(doc, field);
+ termFreqVector = indexReader.getTermVector(doc, field);
if (termFreqVector == null) {
numErrorDocs++;
if (numErrorDocs >= maxErrorDocs) {
@@ -131,8 +153,14 @@ public final class LuceneIterator extend
}
} while (termFreqVector == null);
- indexReader.getTermFreqVector(doc, field, mapper);
- mapper.setDocumentNumber(doc);
+
+ TermsEnum te = termFreqVector.iterator(null);
+ BytesRef term;
+ TFDFMapper mapper = new TFDFMapper(indexReader.numDocs(), weight, this.terminfo);
+ mapper.setExpectations(field, termFreqVector.size());
+ while ((term = te.next()) != null) {
+ mapper.map(term, (int) te.totalTermFreq());
+ }
Vector result = mapper.getVector();
if (result == null) {
// TODO is this right? last version would produce null in the iteration in this case, though it
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/TFDFMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/TFDFMapper.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/TFDFMapper.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/TFDFMapper.java Wed Jan 30 10:27:17 2013
@@ -17,8 +17,7 @@
package org.apache.mahout.utils.vectors.lucene;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermVectorOffsetInfo;
+import org.apache.lucene.util.BytesRef;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.utils.vectors.TermEntry;
@@ -29,50 +28,37 @@ import org.apache.mahout.vectorizer.Weig
/**
* Not thread-safe
*/
-public class TFDFMapper extends VectorMapper {
+public class TFDFMapper {
private Vector vector;
private final Weight weight;
- private int numTerms;
+ private long numTerms;
private final TermInfo termInfo;
private String field;
private final int numDocs;
- public TFDFMapper(IndexReader reader, Weight weight, TermInfo termInfo) {
+ public TFDFMapper(int numDocs, Weight weight, TermInfo termInfo) {
this.weight = weight;
this.termInfo = termInfo;
- this.numDocs = reader.numDocs();
+ this.numDocs = numDocs;
}
-
- @Override
- public Vector getVector() {
- return vector;
- }
-
- @Override
- public void setExpectations(String field, int numTerms, boolean storeOffsets, boolean storePositions) {
+
+ public void setExpectations(String field, long numTerms) {
this.field = field;
vector = new RandomAccessSparseVector(termInfo.totalTerms(field));
this.numTerms = numTerms;
}
- @Override
- public void map(String term, int frequency, TermVectorOffsetInfo[] offsets, int[] positions) {
- TermEntry entry = termInfo.getTermEntry(field, term);
+ public void map(BytesRef term, int frequency) {
+ TermEntry entry = termInfo.getTermEntry(field, term.utf8ToString() );
if (entry != null) {
- vector.setQuick(entry.getTermIdx(), weight.calculate(frequency, entry.getDocFreq(), numTerms, numDocs));
+ vector.setQuick(entry.getTermIdx(), weight.calculate(frequency, entry.getDocFreq(), (int)numTerms, numDocs));
}
}
- @Override
- public boolean isIgnoringPositions() {
- return true;
- }
-
- @Override
- public boolean isIgnoringOffsets() {
- return true;
+ public Vector getVector() {
+ return this.vector;
}
}
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/VectorMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/VectorMapper.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/VectorMapper.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/vectors/lucene/VectorMapper.java Wed Jan 30 10:27:17 2013
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.utils.vectors.lucene;
-
-import org.apache.lucene.index.TermVectorMapper;
-import org.apache.mahout.math.Vector;
-
-/**
- * Not thread-safe
- */
-public abstract class VectorMapper extends TermVectorMapper {
- /**
- * Can be called after the TermVector has been mapped
- * @return The {@link org.apache.mahout.math.Vector}
- *
- * @see #map(String, int, org.apache.lucene.index.TermVectorOffsetInfo[], int[])
- */
- public abstract Vector getVector();
-}
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java?rev=1440334&r1=1440333&r2=1440334&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java Wed Jan 30 10:27:17 2013
@@ -29,8 +29,6 @@ import org.apache.hadoop.io.SequenceFile
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
@@ -64,8 +62,6 @@ import org.apache.mahout.utils.vectors.T
import org.apache.mahout.utils.vectors.TermInfo;
import org.apache.mahout.utils.vectors.lucene.CachedTermInfo;
import org.apache.mahout.utils.vectors.lucene.LuceneIterable;
-import org.apache.mahout.utils.vectors.lucene.TFDFMapper;
-import org.apache.mahout.utils.vectors.lucene.VectorMapper;
import org.apache.mahout.vectorizer.TFIDF;
import org.apache.mahout.vectorizer.Weight;
import org.junit.Before;
@@ -73,6 +69,9 @@ import org.junit.Test;
import com.google.common.collect.Lists;
import com.google.common.io.Closeables;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
public final class TestClusterDumper extends MahoutTestCase {
@@ -112,16 +111,19 @@ public final class TestClusterDumper ext
private void getSampleData(String[] docs2) throws IOException {
sampleData = Lists.newArrayList();
RAMDirectory directory = new RAMDirectory();
- IndexWriter writer = new IndexWriter(directory, new StandardAnalyzer(
- Version.LUCENE_31), true, IndexWriter.MaxFieldLength.UNLIMITED);
+
+ IndexWriter writer = new IndexWriter(directory,
+ new IndexWriterConfig(Version.LUCENE_41,new StandardAnalyzer(
+ Version.LUCENE_41)));
+
try {
for (int i = 0; i < docs2.length; i++) {
Document doc = new Document();
- Fieldable id = new Field("id", "doc_" + i, Field.Store.YES,
+ Field id = new Field("id", "doc_" + i, Field.Store.YES,
Field.Index.NOT_ANALYZED_NO_NORMS);
doc.add(id);
// Store both position and offset information
- Fieldable text = new Field("content", docs2[i], Field.Store.NO,
+ Field text = new Field("content", docs2[i], Field.Store.NO,
Field.Index.ANALYZED, Field.TermVector.YES);
doc.add(text);
writer.addDocument(doc);
@@ -129,7 +131,10 @@ public final class TestClusterDumper ext
} finally {
Closeables.closeQuietly(writer);
}
- IndexReader reader = IndexReader.open(directory, true);
+
+ IndexReader reader = DirectoryReader.open(directory);
+
+
Weight weight = new TFIDF();
TermInfo termInfo = new CachedTermInfo(reader, "content", 1, 100);
@@ -146,9 +151,8 @@ public final class TestClusterDumper ext
System.out.println(i + " " + term);
i++;
}
- VectorMapper mapper = new TFDFMapper(reader, weight, termInfo);
Iterable<Vector> iterable = new LuceneIterable(reader, "id", "content",
- mapper);
+ termInfo,weight);
i = 0;
for (Vector vector : iterable) {