You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by to...@apache.org on 2015/08/03 14:38:25 UTC
svn commit: r1693891 - in /lucene/dev/trunk/lucene/classification/src:
java/org/apache/lucene/classification/ test/org/apache/lucene/classification/
Author: tommaso
Date: Mon Aug 3 12:38:25 2015
New Revision: 1693891
URL: http://svn.apache.org/r1693891
Log:
LUCENE-6654 - KNN taking into consideration class ranking
Modified:
lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
lucene/dev/trunk/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java
lucene/dev/trunk/lucene/classification/src/test/org/apache/lucene/classification/KNearestNeighborClassifierTest.java
Modified: lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java?rev=1693891&r1=1693890&r2=1693891&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java (original)
+++ lucene/dev/trunk/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java Mon Aug 3 12:38:25 2015
@@ -64,7 +64,7 @@ public class KNearestNeighborClassifier
* @param minDocsFreq {@link MoreLikeThis#minDocFreq} parameter
* @param minTermFreq {@link MoreLikeThis#minTermFreq} parameter
* @param classFieldName the name of the field used as the output for the classifier
- * @param textFieldNames the name of the fields used as the inputs for the classifier
+ * @param textFieldNames the name of the fields used as the inputs for the classifier, they can contain boosting indication e.g. title^10
*/
public KNearestNeighborClassifier(LeafReader leafReader, Analyzer analyzer, Query query, int k, int minDocsFreq,
int minTermFreq, String classFieldName, String... textFieldNames) {
@@ -90,17 +90,17 @@ public class KNearestNeighborClassifier
*/
@Override
public ClassificationResult<BytesRef> assignClass(String text) throws IOException {
- TopDocs topDocs = knnSearch(text);
- List<ClassificationResult<BytesRef>> doclist = buildListFromTopDocs(topDocs);
- ClassificationResult<BytesRef> retval = null;
+ TopDocs knnResults = knnSearch(text);
+ List<ClassificationResult<BytesRef>> assignedClasses = buildListFromTopDocs(knnResults);
+ ClassificationResult<BytesRef> assignedClass = null;
double maxscore = -Double.MAX_VALUE;
- for (ClassificationResult<BytesRef> element : doclist) {
- if (element.getScore() > maxscore) {
- retval = element;
- maxscore = element.getScore();
+ for (ClassificationResult<BytesRef> cl : assignedClasses) {
+ if (cl.getScore() > maxscore) {
+ assignedClass = cl;
+ maxscore = cl.getScore();
}
}
- return retval;
+ return assignedClass;
}
/**
@@ -108,10 +108,10 @@ public class KNearestNeighborClassifier
*/
@Override
public List<ClassificationResult<BytesRef>> getClasses(String text) throws IOException {
- TopDocs topDocs = knnSearch(text);
- List<ClassificationResult<BytesRef>> doclist = buildListFromTopDocs(topDocs);
- Collections.sort(doclist);
- return doclist;
+ TopDocs knnResults = knnSearch(text);
+ List<ClassificationResult<BytesRef>> assignedClasses = buildListFromTopDocs(knnResults);
+ Collections.sort(assignedClasses);
+ return assignedClasses;
}
/**
@@ -119,16 +119,27 @@ public class KNearestNeighborClassifier
*/
@Override
public List<ClassificationResult<BytesRef>> getClasses(String text, int max) throws IOException {
- TopDocs topDocs = knnSearch(text);
- List<ClassificationResult<BytesRef>> doclist = buildListFromTopDocs(topDocs);
- Collections.sort(doclist);
- return doclist.subList(0, max);
+ TopDocs knnResults = knnSearch(text);
+ List<ClassificationResult<BytesRef>> assignedClasses = buildListFromTopDocs(knnResults);
+ Collections.sort(assignedClasses);
+ return assignedClasses.subList(0, max);
}
private TopDocs knnSearch(String text) throws IOException {
BooleanQuery.Builder mltQuery = new BooleanQuery.Builder();
- for (String textFieldName : textFieldNames) {
- mltQuery.add(new BooleanClause(mlt.like(textFieldName, new StringReader(text)), BooleanClause.Occur.SHOULD));
+ for (String fieldName : textFieldNames) {
+ String boost = null;
+ mlt.setBoost(true); //terms boost actually helps in MLT queries
+ if (fieldName.contains("^")) {
+ String[] field2boost = fieldName.split("\\^");
+ fieldName = field2boost[0];
+ boost = field2boost[1];
+ }
+ if (boost != null) {
+ mlt.setBoostFactor(Float.parseFloat(boost));//if we have a field boost, we add it
+ }
+ mltQuery.add(new BooleanClause(mlt.like(fieldName, new StringReader(text)), BooleanClause.Occur.SHOULD));
+ mlt.setBoostFactor(1);// restore neutral boost for next field
}
Query classFieldQuery = new WildcardQuery(new Term(classFieldName, "*"));
mltQuery.add(new BooleanClause(classFieldQuery, BooleanClause.Occur.MUST));
@@ -138,18 +149,30 @@ public class KNearestNeighborClassifier
return indexSearcher.search(mltQuery.build(), k);
}
+ //ranking of classes must be taken in consideration
private List<ClassificationResult<BytesRef>> buildListFromTopDocs(TopDocs topDocs) throws IOException {
Map<BytesRef, Integer> classCounts = new HashMap<>();
+ Map<BytesRef, Double> classBoosts = new HashMap<>(); // this is a boost based on class ranking positions in topDocs
+ float maxScore = topDocs.getMaxScore();
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
StorableField storableField = indexSearcher.doc(scoreDoc.doc).getField(classFieldName);
if (storableField != null) {
BytesRef cl = new BytesRef(storableField.stringValue());
+ //update count
Integer count = classCounts.get(cl);
if (count != null) {
classCounts.put(cl, count + 1);
} else {
classCounts.put(cl, 1);
}
+ //update boost, the boost is based on the best score
+ Double totalBoost = classBoosts.get(cl);
+ double singleBoost = scoreDoc.score / maxScore;
+ if (totalBoost != null) {
+ classBoosts.put(cl, totalBoost + singleBoost);
+ } else {
+ classBoosts.put(cl, singleBoost);
+ }
}
}
List<ClassificationResult<BytesRef>> returnList = new ArrayList<>();
@@ -157,7 +180,8 @@ public class KNearestNeighborClassifier
int sumdoc = 0;
for (Map.Entry<BytesRef, Integer> entry : classCounts.entrySet()) {
Integer count = entry.getValue();
- temporaryList.add(new ClassificationResult<>(entry.getKey().clone(), count / (double) k));
+ Double normBoost = classBoosts.get(entry.getKey()) / count; //the boost is normalized to be 0<b<1
+ temporaryList.add(new ClassificationResult<>(entry.getKey().clone(), (count * normBoost) / (double) k));
sumdoc += count;
}
@@ -171,5 +195,4 @@ public class KNearestNeighborClassifier
}
return returnList;
}
-
}
Modified: lucene/dev/trunk/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java?rev=1693891&r1=1693890&r2=1693891&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java (original)
+++ lucene/dev/trunk/lucene/classification/src/test/org/apache/lucene/classification/ClassificationTestBase.java Mon Aug 3 12:38:25 2015
@@ -46,6 +46,14 @@ public abstract class ClassificationTest
public static final String TECHNOLOGY_INPUT = "Much is made of what the likes of Facebook, Google and Apple know about users." +
" Truth is, Amazon may know more.";
+
+ public static final String STRONG_TECHNOLOGY_INPUT = "Much is made of what the likes of Facebook, Google and Apple know about users." +
+ " Truth is, Amazon may know more. This technology observation is extracted from the internet.";
+
+ public static final String SUPER_STRONG_TECHNOLOGY_INPUT = "More than 400 million people trust Google with their e-mail, and 50 million store files" +
+ " in the cloud using the Dropbox service. People manage their bank accounts, pay bills, trade stocks and " +
+ "generally transfer or store huge volumes of personal data online. traveling seeks raises some questions Republican presidential. ";
+
public static final BytesRef TECHNOLOGY_RESULT = new BytesRef("technology");
protected RandomIndexWriter indexWriter;
Modified: lucene/dev/trunk/lucene/classification/src/test/org/apache/lucene/classification/KNearestNeighborClassifierTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/classification/src/test/org/apache/lucene/classification/KNearestNeighborClassifierTest.java?rev=1693891&r1=1693890&r2=1693891&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/classification/src/test/org/apache/lucene/classification/KNearestNeighborClassifierTest.java (original)
+++ lucene/dev/trunk/lucene/classification/src/test/org/apache/lucene/classification/KNearestNeighborClassifierTest.java Mon Aug 3 12:38:25 2015
@@ -16,9 +16,12 @@
*/
package org.apache.lucene.classification;
+import java.util.List;
+
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
@@ -40,6 +43,55 @@ public class KNearestNeighborClassifierT
} finally {
if (leafReader != null) {
leafReader.close();
+ }
+ }
+ }
+
+ /**
+ * This test is for the scenario where in the first topK results from the MLT query, we have the same number of results per class.
+ * But the results for a class have a better ranking in comparison with the results of the second class.
+ * So we would expect a greater score for the best ranked class.
+ *
+ * @throws Exception if any error happens
+ */
+ @Test
+ public void testRankedClasses() throws Exception {
+ LeafReader leafReader = null;
+ try {
+ Analyzer analyzer = new EnglishAnalyzer();
+ leafReader = populateSampleIndex(analyzer);
+ KNearestNeighborClassifier knnClassifier = new KNearestNeighborClassifier(leafReader, analyzer, null, 6, 1, 1, categoryFieldName, textFieldName);
+ List<ClassificationResult<BytesRef>> classes = knnClassifier.getClasses(STRONG_TECHNOLOGY_INPUT);
+ assertTrue(classes.get(0).getScore() > classes.get(1).getScore());
+ checkCorrectClassification(knnClassifier, STRONG_TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
+ } finally {
+ if (leafReader != null) {
+ leafReader.close();
+ }
+ }
+ }
+
+ /**
+ * This test is for the scenario where in the first topK results from the MLT query, we have less results
+ * for the expected class than the results for the bad class.
+ * But the results for the expected class have a better score in comparison with the results of the second class.
+ * So we would expect a greater score for the best ranked class.
+ *
+ * @throws Exception if any error happens
+ */
+ @Test
+ public void testUnbalancedClasses() throws Exception {
+ LeafReader leafReader = null;
+ try {
+ Analyzer analyzer = new EnglishAnalyzer();
+ leafReader = populateSampleIndex(analyzer);
+ KNearestNeighborClassifier knnClassifier = new KNearestNeighborClassifier(leafReader, analyzer, null, 3, 1, 1, categoryFieldName, textFieldName);
+ List<ClassificationResult<BytesRef>> classes = knnClassifier.getClasses(SUPER_STRONG_TECHNOLOGY_INPUT);
+ assertTrue(classes.get(0).getScore() > classes.get(1).getScore());
+ checkCorrectClassification(knnClassifier, SUPER_STRONG_TECHNOLOGY_INPUT, TECHNOLOGY_RESULT);
+ } finally {
+ if (leafReader != null) {
+ leafReader.close();
}
}
}