You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/07/03 02:43:07 UTC

svn commit: r1356531 [3/3] - in /lucene/dev/branches/pforcodec_3892/lucene: core/src/java/org/apache/lucene/codecs/pfor/ core/src/resources/META-INF/services/ core/src/test/org/apache/lucene/codecs/pfor/ core/src/test/org/apache/lucene/index/ test-fram...

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pfor/gendecompress.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pfor/gendecompress.py?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pfor/gendecompress.py (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pfor/gendecompress.py Tue Jul  3 00:43:06 2012
@@ -21,7 +21,6 @@ Generate source code for java classes fo
 """
 
 USE_SCRATCH = False
-#USE_SCRATCH = True 
 
 def bitsExpr(i, numFrameBits):
   framePos = i * numFrameBits
@@ -51,7 +50,7 @@ def bitsExpr(i, numFrameBits):
 
 
 def genDecompress():
-  className = "ForDecompressImpl"
+  className = "PackedIntsDecompress"
   fileName = className + ".java"
   imports = "import java.nio.IntBuffer;\n"
   f = open(fileName, 'w')
@@ -80,7 +79,7 @@ def genDecompress():
 
     w("import java.nio.IntBuffer;\n\n")
 
-    w("final class ForDecompressImpl {\n")
+    w("final class PackedIntsDecompress {\n")
 
     w('\n  // nocommit: assess perf of this to see if specializing is really needed\n')
 
@@ -118,7 +117,7 @@ def genDecompress():
 
 def genSwitch():
   for numFrameBits in xrange(1, 33):
-    print '      case %d: ForDecompressImpl.decode%d(compressedBuffer, encoded); break;' % (numFrameBits, numFrameBits)
+    print '      case %d: PackedIntsDecompress.decode%d(compressedBuffer, encoded); break;' % (numFrameBits, numFrameBits)
 
 if __name__ == "__main__":
   genDecompress()

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Tue Jul  3 00:43:06 2012
@@ -18,3 +18,4 @@ org.apache.lucene.codecs.pulsing.Pulsing
 org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat
 org.apache.lucene.codecs.memory.MemoryPostingsFormat
 org.apache.lucene.codecs.pfor.ForPostingsFormat
+org.apache.lucene.codecs.pfor.PForPostingsFormat

Added: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java?rev=1356531&view=auto
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java (added)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java Tue Jul  3 00:43:06 2012
@@ -0,0 +1,162 @@
+package org.apache.lucene.codecs.pfor;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.*;
+import java.io.*;
+import java.nio.*;
+import org.apache.lucene.codecs.pfor.*;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestPForUtil extends LuceneTestCase {
+  static final int[] MASK={ 0x00000000,
+    0x00000001, 0x00000003, 0x00000007, 0x0000000f, 0x0000001f, 0x0000003f,
+    0x0000007f, 0x000000ff, 0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff,
+    0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff, 0x0001ffff, 0x0003ffff,
+    0x0007ffff, 0x000fffff, 0x001fffff, 0x003fffff, 0x007fffff, 0x00ffffff,
+    0x01ffffff, 0x03ffffff, 0x07ffffff, 0x0fffffff, 0x1fffffff, 0x3fffffff,
+    0x7fffffff, 0xffffffff};
+  Random gen;
+  long seed=System.currentTimeMillis();
+  //long seed=1338528171959L;
+  public void initRandom() {
+  //  println("Seed: "+seed);
+    this.gen = new Random(seed);
+  }
+  public void testCompress() throws Exception {
+    initRandom();
+    tryForcedException();
+    tryAllDistribution();
+  }
+
+  // Test correctness of ignored forced exception
+  public void tryForcedException() throws Exception {
+    int sz=128;
+    Integer[] buff= new Integer[sz];
+    int[] data = new int[sz];
+    int[] copy = new int[sz];
+    byte[] res = new byte[4+sz*8];
+    IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer();
+    for (int i=0; i<sz-1; ++i)
+      buff[i]=gen.nextInt() & 1;
+    buff[sz-1]=gen.nextInt() & 0xffffffff;   // create only one exception
+
+    Collections.shuffle(Arrays.asList(buff),new Random(seed));
+    for (int i=0; i<sz; ++i)
+      data[i] = buff[i];
+
+    int ensz = PForUtil.compress(data,sz,resBuffer);
+
+    if (ensz > sz*8+4) {
+      println("Excceed? "+ensz+">"+(sz*8+4));
+      ensz=sz*8+4;
+    }
+    resBuffer.rewind();
+    PForUtil.decompress(resBuffer,copy);
+
+    //println(getHex(data,sz)+"\n");
+    //println(getHex(res,ensz)+"\n");
+    //println(getHex(copy,sz)+"\n");
+    
+    assert cmp(data,sz,copy,sz)==true;
+  }
+
+  // Test correctness of compressing and decompressing
+  public void tryAllDistribution() throws Exception {
+    for (int i=0; i<=32; ++i) { // try to test every kinds of distribution
+      double alpha=gen.nextDouble(); // rate of normal value
+      for (int j=0; j<=32; ++j) {
+        tryDistribution(128,alpha,MASK[i],MASK[j]);
+      }
+    }
+  }
+  public void tryDistribution(int sz, double alpha, int masknorm, int maskexc) throws Exception {
+    Integer[] buff= new Integer[sz];
+    int[] data = new int[sz];
+    byte[] res = new byte[4+sz*8];      // loosely upperbound
+    IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer();
+    int i=0;
+    for (; i<sz*alpha; ++i)
+      buff[i]=gen.nextInt() & masknorm;
+    for (; i<sz; ++i)
+      buff[i]=gen.nextInt() & maskexc;
+    Collections.shuffle(Arrays.asList(buff),new Random(seed));
+    for (i=0; i<sz; ++i)
+      data[i] = buff[i];
+
+    int ensz = PForUtil.compress(data,sz,resBuffer);
+    
+    if (ensz > sz*8+4) {
+      println("Excceed? "+ensz+">"+(sz*8+4));
+      ensz=sz*8+4;
+    }
+    int[] copy = new int[sz];
+    PForUtil.decompress(resBuffer,copy);
+
+//    println(getHex(data,sz)+"\n");
+//    println(getHex(res,ensz)+"\n");
+//    println(getHex(copy,sz)+"\n");
+
+    assert cmp(data,sz,copy,sz)==true;
+  }
+  public boolean cmp(int[] a, int sza, int[] b, int szb) {
+    if (sza!=szb)
+      return false;
+    for (int i=0; i<sza; ++i) {
+      if (a[i]!=b[i]) {
+        System.err.println(String.format("! %08x != %08x in %d",a[i],b[i],i));
+        return false;
+      }
+    }
+    return true;
+  }
+  public static String getHex( byte [] raw, int sz ) {
+    final String HEXES = "0123456789ABCDEF";
+    if ( raw == null ) {
+      return null;
+    }
+    final StringBuilder hex = new StringBuilder( 2 * raw.length );
+    for ( int i=0; i<sz; i++ ) {
+      if (i>0 && (i)%16 == 0)
+        hex.append("\n");
+      byte b=raw[i];
+      hex.append(HEXES.charAt((b & 0xF0) >> 4))
+         .append(HEXES.charAt((b & 0x0F)))
+         .append(" ");
+    }
+    return hex.toString();
+  }
+  public static String getHex( int [] raw, int sz ) {
+    if ( raw == null ) {
+      return null;
+    }
+    final StringBuilder hex = new StringBuilder( 4 * raw.length );
+    for ( int i=0; i<sz; i++ ) {
+      if (i>0 && i%8 == 0)
+        hex.append("\n");
+      hex.append(String.format("%08x ",raw[i]));
+    }
+    return hex.toString();
+  }
+  static void println(String format, Object... args) {
+    System.out.println(String.format(format,args)); 
+  }
+  static void print(String format, Object... args) {
+    System.out.print(String.format(format,args)); 
+  }
+}

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Tue Jul  3 00:43:06 2012
@@ -76,7 +76,7 @@ import org.junit.BeforeClass;
 // we won't even be running the actual code, only the impostor
 // @SuppressCodecs("Lucene4x")
 // Sep codec cannot yet handle the offsets in our 4.x index!
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom","For"})
+@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom","For","PFor"})
 public class TestBackwardsCompatibility extends LuceneTestCase {
 
   // Uncomment these cases & run them on an older Lucene

Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java Tue Jul  3 00:43:06 2012
@@ -49,7 +49,7 @@ import org.apache.lucene.util._TestUtil;
 // TODO: we really need to test indexingoffsets, but then getting only docs / docs + freqs.
 // not all codecs store prx separate...
 // TODO: fix sep codec to index offsets so we can greatly reduce this list!
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom","For"})
+@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom","For","PFor"})
 public class TestPostingsOffsets extends LuceneTestCase {
   IndexWriterConfig iwc;
   

Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Tue Jul  3 00:43:06 2012
@@ -487,6 +487,7 @@ public abstract class BaseTokenStreamTes
     add("MockSep");
     add("MockRandom");
     add("For");
+    add("PFor");
   }};
   
   private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect, RandomIndexWriter iw) throws IOException {

Modified: lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1356531&r1=1356530&r2=1356531&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Tue Jul  3 00:43:06 2012
@@ -95,6 +95,7 @@ public class MockRandomPostingsFormat ex
       delegates.add(new MockVariableIntBlockPostingsFormat.MockIntFactory(baseBlockSize));
       // TODO: others
       delegates.add(new ForFactory());
+      delegates.add(new PForFactory());
     }
 
     private static String getExtension(String fileName) {