You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/11/07 03:59:00 UTC
svn commit: r1406428 - in /lucene/dev/trunk/lucene: CHANGES.txt
core/src/java/org/apache/lucene/index/Norm.java
core/src/test/org/apache/lucene/index/TestCustomNorms.java
Author: rmuir
Date: Wed Nov 7 02:59:00 2012
New Revision: 1406428
URL: http://svn.apache.org/viewvc?rev=1406428&view=rev
Log:
LUCENE-4540: allow packed ints norms
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/Norm.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1406428&r1=1406427&r2=1406428&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Wed Nov 7 02:59:00 2012
@@ -72,6 +72,11 @@ New Features
* LUCENE-4515: MemoryIndex now supports adding the same field multiple
times. (Simon Willnauer)
+* LUCENE-4540: Added an experimental Norm.setPackedLong, which allows
+ the use of VAR_INTS-encoded norms. This can be useful for cases where
+ you only need a few bits per-document, or where you might want exact
+ document length, and so on. (Robert Muir)
+
API Changes
* LUCENE-4399: Deprecated AppendingCodec. Lucene's term dictionaries
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/Norm.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/Norm.java?rev=1406428&r1=1406427&r2=1406428&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/Norm.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/Norm.java Wed Nov 7 02:59:00 2012
@@ -115,6 +115,15 @@ public final class Norm {
setType(Type.FIXED_INTS_64);
this.field.setLongValue(norm);
}
+
+ /**
+ * Sets a packed long norm value.
+ * @lucene.experimental
+ */
+ public void setPackedLong(long norm) {
+ setType(Type.VAR_INTS);
+ this.field.setLongValue(norm);
+ }
/**
* Sets a byte norm value
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java?rev=1406428&r1=1406427&r2=1406428&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java Wed Nov 7 02:59:00 2012
@@ -22,6 +22,7 @@ import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues.Type;
@@ -30,14 +31,12 @@ import org.apache.lucene.search.TermStat
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.search.similarities.Similarity.ExactSimScorer;
-import org.apache.lucene.search.similarities.Similarity.SimWeight;
-import org.apache.lucene.search.similarities.Similarity.SloppySimScorer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
/**
*
@@ -87,6 +86,39 @@ public class TestCustomNorms extends Luc
dir.close();
docs.close();
}
+
+ public void testPackedNorms() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ config.setSimilarity(new PackedNormSimilarity());
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
+ int num = _TestUtil.nextInt(random(), 1, 1000);
+ for (int i = 0; i < num; i++) {
+ Document doc = new Document();
+ doc.add(new StringField("len", Integer.toString(i), Field.Store.YES));
+ StringBuilder sb = new StringBuilder();
+ for (int j = 0; j < i; j++) {
+ sb.append(" token");
+ }
+ doc.add(new TextField("content", sb.toString(), Field.Store.NO));
+ writer.addDocument(doc);
+ }
+
+ DirectoryReader ir = writer.getReader();
+ writer.close();
+ for (AtomicReaderContext context : ir.leaves()) {
+ AtomicReader reader = context.reader();
+ DocValues norms = reader.normValues("content");
+ assertNotNull(norms);
+ Source source = norms.getSource();
+ assertEquals(Type.VAR_INTS, source.getType());
+ for (int i = 0; i < reader.maxDoc(); i++) {
+ assertEquals(source.getInt(i), Long.parseLong(reader.document(i).get("len")));
+ }
+ }
+ ir.close();
+ dir.close();
+ }
public void testExceptionOnRandomType() throws IOException {
Directory dir = newDirectory();
@@ -302,5 +334,28 @@ public class TestCustomNorms extends Luc
throw new UnsupportedOperationException();
}
}
+
+ class PackedNormSimilarity extends Similarity {
+
+ @Override
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ norm.setPackedLong(state.getLength());
+ }
+
+ @Override
+ public SimWeight computeWeight(float queryBoost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ExactSimScorer exactSimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SloppySimScorer sloppySimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+ }
}