You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/02/12 06:43:23 UTC

[33/50] incubator-kylin git commit: drop compression feature inside HLLC SerDes

drop compression feature inside HLLC SerDes


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8d84c1f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8d84c1f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8d84c1f1

Branch: refs/heads/inverted-index
Commit: 8d84c1f11dd5809ab01624c3c9c1a96346d8e78e
Parents: 8466b53
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Feb 11 15:49:35 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Feb 11 15:49:35 2015 +0800

----------------------------------------------------------------------
 .../common/hll/HyperLogLogPlusCounter.java      | 76 +-------------------
 1 file changed, 1 insertion(+), 75 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d84c1f1/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index 18b3af1..3bcae86 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -18,22 +18,15 @@
 
 package org.apache.kylin.common.hll;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.util.Arrays;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
 
-import org.apache.commons.compress.utils.IOUtils;
+import org.apache.kylin.common.util.BytesUtil;
 
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
-import org.apache.kylin.common.util.BytesUtil;
-import com.ning.compress.lzf.LZFDecoder;
-import com.ning.compress.lzf.LZFEncoder;
 
 /**
  * About compression, test on HLLC data shows
@@ -168,54 +161,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
 
     // ============================================================================
 
-    public static interface Compressor {
-
-        byte[] compress(ByteBuffer buf, int offset, int length) throws IOException;
-
-        byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException;
-    }
-
-    static final Compressor GZIP_COMPRESSOR = new Compressor() {
-        @Override
-        public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
-            ByteArrayOutputStream bout = new ByteArrayOutputStream();
-            GZIPOutputStream gzout = new GZIPOutputStream(bout);
-            gzout.write(buf.array(), offset, length);
-            gzout.close();
-            return bout.toByteArray();
-        }
-
-        @Override
-        public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
-            ByteArrayInputStream bin = new ByteArrayInputStream(buf.array(), offset, length);
-            GZIPInputStream gzin = new GZIPInputStream(bin);
-            ByteArrayOutputStream bout = new ByteArrayOutputStream();
-            IOUtils.copy(gzin, bout);
-            gzin.close();
-            bout.close();
-            return bout.toByteArray();
-        }
-    };
-
-    static final Compressor LZF_COMPRESSOR = new Compressor() {
-        @Override
-        public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
-            return LZFEncoder.encode(buf.array(), offset, length);
-        }
-
-        @Override
-        public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
-            return LZFDecoder.decode(buf.array(), offset, length);
-        }
-    };
-
-    public static final int COMPRESSION_THRESHOLD = Integer.MAX_VALUE; // bytes, disable due to slowness
-    public static final byte COMPRESSION_FLAG = (byte) 0x02;
-    public static final Compressor DEFAULT_COMPRESSOR = GZIP_COMPRESSOR; // LZF lib has a bug at the moment
-
     public void writeRegisters(final ByteBuffer out) throws IOException {
-        int startPos = out.position();
-
         final int indexLen = getRegisterIndexSize();
         int size = size();
 
@@ -240,30 +186,10 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
                 out.put(registers[i]);
             }
         }
-
-        // do compression if needed
-        int len = out.position() - startPos;
-        if (len < COMPRESSION_THRESHOLD)
-            return;
-
-        scheme |= COMPRESSION_FLAG;
-        byte[] compressed = DEFAULT_COMPRESSOR.compress(out, startPos + 1, len - 1);
-        out.position(startPos);
-        out.put(scheme);
-        BytesUtil.writeVInt(compressed.length, out);
-        out.put(compressed);
     }
 
     public void readRegisters(ByteBuffer in) throws IOException {
         byte scheme = in.get();
-        if ((scheme & COMPRESSION_FLAG) > 0) {
-            scheme ^= COMPRESSION_FLAG;
-            int compressedLen = BytesUtil.readVInt(in);
-            int end = in.position() + compressedLen;
-            byte[] decompressed = DEFAULT_COMPRESSOR.decompress(in, in.position(), compressedLen);
-            in.position(end);
-            in = ByteBuffer.wrap(decompressed);
-        }
 
         if (scheme == 0) { // map scheme
             clear();