You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/06/10 14:48:03 UTC
svn commit: r1601625 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/codecs/ lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/
lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/ lucene/core/
lucene/core/src/java/org/apache/lucene/...
Author: rmuir
Date: Tue Jun 10 12:48:02 2014
New Revision: 1601625
URL: http://svn.apache.org/r1601625
Log:
LUCENE-5743: Add Lucene49NormsFormat
Added:
lucene/dev/branches/branch_4x/lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestDiskNormsFormat.java
- copied unchanged from r1601606, lucene/dev/trunk/lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestDiskNormsFormat.java
lucene/dev/branches/branch_4x/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextNormsFormat.java
- copied unchanged from r1601606, lucene/dev/trunk/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextNormsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsConsumer.java
- copied unchanged from r1601606, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsConsumer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
- copied unchanged from r1601606, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
- copied unchanged from r1601606, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java
- copied unchanged from r1601606, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java
- copied unchanged from r1601606, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java
- copied unchanged from r1601606, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNormsFormat.java
- copied, changed from r1601606, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNormsFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
- copied, changed from r1601606, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1601625&r1=1601624&r2=1601625&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Tue Jun 10 12:48:02 2014
@@ -38,6 +38,9 @@ New Features
from Directory. Add Lucene49Codec and Lucene49DocValuesFormat that make
use of these. (Robert Muir)
+* LUCENE-5743: Add Lucene49NormsFormat, which can compress in some cases
+ such as very short fields. (Ryan Ernst, Adrien Grand, Robert Muir)
+
Changes in Backwards Compatibility Policy
* LUCENE-5634: Add reuse argument to IndexableField.tokenStream. This
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java?rev=1601625&r1=1601624&r2=1601625&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49Codec.java Tue Jun 10 12:48:02 2014
@@ -131,7 +131,7 @@ public class Lucene49Codec extends Codec
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene49");
- private final NormsFormat normsFormat = new Lucene42NormsFormat();
+ private final NormsFormat normsFormat = new Lucene49NormsFormat();
@Override
public final NormsFormat normsFormat() {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java?rev=1601625&r1=1601624&r2=1601625&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java Tue Jun 10 12:48:02 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene4
import java.io.Closeable; // javadocs
import java.io.IOException;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -199,6 +200,7 @@ public class Lucene49DocValuesConsumer e
break;
case TABLE_COMPRESSED:
final Long[] decode = uniqueValues.toArray(new Long[uniqueValues.size()]);
+ Arrays.sort(decode);
final HashMap<Long,Integer> encode = new HashMap<>();
meta.writeVInt(decode.length);
for (int i = 0; i < decode.length; i++) {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html?rev=1601625&r1=1601624&r2=1601625&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene49/package.html Tue Jun 10 12:48:02 2014
@@ -173,7 +173,7 @@ term occurs in each document. Note that
all documents omit position data.
</li>
<li>
-{@link org.apache.lucene.codecs.lucene42.Lucene42NormsFormat Normalization factors}.
+{@link org.apache.lucene.codecs.lucene49.Lucene49NormsFormat Normalization factors}.
For each field in each document, a value is stored
that is multiplied into the score for hits on that field.
</li>
@@ -289,7 +289,7 @@ systems that frequently run out of file
<td>Stores additional per-position metadata information such as character offsets and user payloads</td>
</tr>
<tr>
-<td>{@link org.apache.lucene.codecs.lucene42.Lucene42NormsFormat Norms}</td>
+<td>{@link org.apache.lucene.codecs.lucene49.Lucene49NormsFormat Norms}</td>
<td>.nvd, .nvm</td>
<td>Encodes length and boost factors for docs and fields</td>
</tr>
Copied: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNormsFormat.java (from r1601606, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNormsFormat.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNormsFormat.java?p2=lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNormsFormat.java&p1=lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNormsFormat.java&r1=1601606&r2=1601625&rev=1601625&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestNormsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestNormsFormat.java Tue Jun 10 12:48:02 2014
@@ -18,9 +18,11 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
/** Tests the codec configuration defined by LuceneTestCase randomly
*/
+@SuppressCodecs("Lucene3x") // 3.x only supports single byte norms
public class TestNormsFormat extends BaseNormsFormatTestCase {
@Override
Copied: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java (from r1601606, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java?p2=lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java&p1=lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java&r1=1601606&r2=1601625&rev=1601625&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java Tue Jun 10 12:48:02 2014
@@ -130,7 +130,7 @@ public abstract class BaseNormsFormatTes
writer.deleteDocuments(new Term("id", Integer.toString(id)));
}
- writer.shutdown();
+ writer.close();
// compare
DirectoryReader ir = DirectoryReader.open(dir);