You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/07/03 02:43:07 UTC
svn commit: r1356531 [3/3] - in /lucene/dev/branches/pforcodec_3892/lucene:
core/src/java/org/apache/lucene/codecs/pfor/
core/src/resources/META-INF/services/
core/src/test/org/apache/lucene/codecs/pfor/
core/src/test/org/apache/lucene/index/ test-fram...
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pfor/gendecompress.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pfor/gendecompress.py?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pfor/gendecompress.py (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pfor/gendecompress.py Tue Jul 3 00:43:06 2012
@@ -21,7 +21,6 @@ Generate source code for java classes fo
"""
USE_SCRATCH = False
-#USE_SCRATCH = True
def bitsExpr(i, numFrameBits):
framePos = i * numFrameBits
@@ -51,7 +50,7 @@ def bitsExpr(i, numFrameBits):
def genDecompress():
- className = "ForDecompressImpl"
+ className = "PackedIntsDecompress"
fileName = className + ".java"
imports = "import java.nio.IntBuffer;\n"
f = open(fileName, 'w')
@@ -80,7 +79,7 @@ def genDecompress():
w("import java.nio.IntBuffer;\n\n")
- w("final class ForDecompressImpl {\n")
+ w("final class PackedIntsDecompress {\n")
w('\n // nocommit: assess perf of this to see if specializing is really needed\n')
@@ -118,7 +117,7 @@ def genDecompress():
def genSwitch():
for numFrameBits in xrange(1, 33):
- print ' case %d: ForDecompressImpl.decode%d(compressedBuffer, encoded); break;' % (numFrameBits, numFrameBits)
+ print ' case %d: PackedIntsDecompress.decode%d(compressedBuffer, encoded); break;' % (numFrameBits, numFrameBits)
if __name__ == "__main__":
genDecompress()
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Tue Jul 3 00:43:06 2012
@@ -18,3 +18,4 @@ org.apache.lucene.codecs.pulsing.Pulsing
org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat
org.apache.lucene.codecs.memory.MemoryPostingsFormat
org.apache.lucene.codecs.pfor.ForPostingsFormat
+org.apache.lucene.codecs.pfor.PForPostingsFormat
Added: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java?rev=1356531&view=auto
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java (added)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java Tue Jul 3 00:43:06 2012
@@ -0,0 +1,162 @@
+package org.apache.lucene.codecs.pfor;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.*;
+import java.io.*;
+import java.nio.*;
+import org.apache.lucene.codecs.pfor.*;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestPForUtil extends LuceneTestCase {
+ static final int[] MASK={ 0x00000000,
+ 0x00000001, 0x00000003, 0x00000007, 0x0000000f, 0x0000001f, 0x0000003f,
+ 0x0000007f, 0x000000ff, 0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff,
+ 0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff, 0x0001ffff, 0x0003ffff,
+ 0x0007ffff, 0x000fffff, 0x001fffff, 0x003fffff, 0x007fffff, 0x00ffffff,
+ 0x01ffffff, 0x03ffffff, 0x07ffffff, 0x0fffffff, 0x1fffffff, 0x3fffffff,
+ 0x7fffffff, 0xffffffff};
+ Random gen;
+ long seed=System.currentTimeMillis();
+ //long seed=1338528171959L;
+ public void initRandom() {
+ // println("Seed: "+seed);
+ this.gen = new Random(seed);
+ }
+ public void testCompress() throws Exception {
+ initRandom();
+ tryForcedException();
+ tryAllDistribution();
+ }
+
+ // Test correctness of ignored forced exception
+ public void tryForcedException() throws Exception {
+ int sz=128;
+ Integer[] buff= new Integer[sz];
+ int[] data = new int[sz];
+ int[] copy = new int[sz];
+ byte[] res = new byte[4+sz*8];
+ IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer();
+ for (int i=0; i<sz-1; ++i)
+ buff[i]=gen.nextInt() & 1;
+ buff[sz-1]=gen.nextInt() & 0xffffffff; // create only one exception
+
+ Collections.shuffle(Arrays.asList(buff),new Random(seed));
+ for (int i=0; i<sz; ++i)
+ data[i] = buff[i];
+
+ int ensz = PForUtil.compress(data,sz,resBuffer);
+
+ if (ensz > sz*8+4) {
+ println("Excceed? "+ensz+">"+(sz*8+4));
+ ensz=sz*8+4;
+ }
+ resBuffer.rewind();
+ PForUtil.decompress(resBuffer,copy);
+
+ //println(getHex(data,sz)+"\n");
+ //println(getHex(res,ensz)+"\n");
+ //println(getHex(copy,sz)+"\n");
+
+ assert cmp(data,sz,copy,sz)==true;
+ }
+
+ // Test correctness of compressing and decompressing
+ public void tryAllDistribution() throws Exception {
+ for (int i=0; i<=32; ++i) { // try to test every kinds of distribution
+ double alpha=gen.nextDouble(); // rate of normal value
+ for (int j=0; j<=32; ++j) {
+ tryDistribution(128,alpha,MASK[i],MASK[j]);
+ }
+ }
+ }
+ public void tryDistribution(int sz, double alpha, int masknorm, int maskexc) throws Exception {
+ Integer[] buff= new Integer[sz];
+ int[] data = new int[sz];
+ byte[] res = new byte[4+sz*8]; // loosely upperbound
+ IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer();
+ int i=0;
+ for (; i<sz*alpha; ++i)
+ buff[i]=gen.nextInt() & masknorm;
+ for (; i<sz; ++i)
+ buff[i]=gen.nextInt() & maskexc;
+ Collections.shuffle(Arrays.asList(buff),new Random(seed));
+ for (i=0; i<sz; ++i)
+ data[i] = buff[i];
+
+ int ensz = PForUtil.compress(data,sz,resBuffer);
+
+ if (ensz > sz*8+4) {
+ println("Excceed? "+ensz+">"+(sz*8+4));
+ ensz=sz*8+4;
+ }
+ int[] copy = new int[sz];
+ PForUtil.decompress(resBuffer,copy);
+
+// println(getHex(data,sz)+"\n");
+// println(getHex(res,ensz)+"\n");
+// println(getHex(copy,sz)+"\n");
+
+ assert cmp(data,sz,copy,sz)==true;
+ }
+ public boolean cmp(int[] a, int sza, int[] b, int szb) {
+ if (sza!=szb)
+ return false;
+ for (int i=0; i<sza; ++i) {
+ if (a[i]!=b[i]) {
+ System.err.println(String.format("! %08x != %08x in %d",a[i],b[i],i));
+ return false;
+ }
+ }
+ return true;
+ }
+ public static String getHex( byte [] raw, int sz ) {
+ final String HEXES = "0123456789ABCDEF";
+ if ( raw == null ) {
+ return null;
+ }
+ final StringBuilder hex = new StringBuilder( 2 * raw.length );
+ for ( int i=0; i<sz; i++ ) {
+ if (i>0 && (i)%16 == 0)
+ hex.append("\n");
+ byte b=raw[i];
+ hex.append(HEXES.charAt((b & 0xF0) >> 4))
+ .append(HEXES.charAt((b & 0x0F)))
+ .append(" ");
+ }
+ return hex.toString();
+ }
+ public static String getHex( int [] raw, int sz ) {
+ if ( raw == null ) {
+ return null;
+ }
+ final StringBuilder hex = new StringBuilder( 4 * raw.length );
+ for ( int i=0; i<sz; i++ ) {
+ if (i>0 && i%8 == 0)
+ hex.append("\n");
+ hex.append(String.format("%08x ",raw[i]));
+ }
+ return hex.toString();
+ }
+ static void println(String format, Object... args) {
+ System.out.println(String.format(format,args));
+ }
+ static void print(String format, Object... args) {
+ System.out.print(String.format(format,args));
+ }
+}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Tue Jul 3 00:43:06 2012
@@ -76,7 +76,7 @@ import org.junit.BeforeClass;
// we won't even be running the actual code, only the impostor
// @SuppressCodecs("Lucene4x")
// Sep codec cannot yet handle the offsets in our 4.x index!
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom","For"})
+@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom","For","PFor"})
public class TestBackwardsCompatibility extends LuceneTestCase {
// Uncomment these cases & run them on an older Lucene
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java Tue Jul 3 00:43:06 2012
@@ -49,7 +49,7 @@ import org.apache.lucene.util._TestUtil;
// TODO: we really need to test indexingoffsets, but then getting only docs / docs + freqs.
// not all codecs store prx separate...
// TODO: fix sep codec to index offsets so we can greatly reduce this list!
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom","For"})
+@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom","For","PFor"})
public class TestPostingsOffsets extends LuceneTestCase {
IndexWriterConfig iwc;
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Tue Jul 3 00:43:06 2012
@@ -487,6 +487,7 @@ public abstract class BaseTokenStreamTes
add("MockSep");
add("MockRandom");
add("For");
+ add("PFor");
}};
private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect, RandomIndexWriter iw) throws IOException {
Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Tue Jul 3 00:43:06 2012
@@ -95,6 +95,7 @@ public class MockRandomPostingsFormat ex
delegates.add(new MockVariableIntBlockPostingsFormat.MockIntFactory(baseBlockSize));
// TODO: others
delegates.add(new ForFactory());
+ delegates.add(new PForFactory());
}
private static String getExtension(String fileName) {