You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/02/12 06:43:23 UTC
[33/50] incubator-kylin git commit: drop compression feature inside
HLLC SerDes
drop compression feature inside HLLC SerDes
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8d84c1f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8d84c1f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8d84c1f1
Branch: refs/heads/inverted-index
Commit: 8d84c1f11dd5809ab01624c3c9c1a96346d8e78e
Parents: 8466b53
Author: Li, Yang <ya...@ebay.com>
Authored: Wed Feb 11 15:49:35 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Wed Feb 11 15:49:35 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 76 +-------------------
1 file changed, 1 insertion(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d84c1f1/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index 18b3af1..3bcae86 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -18,22 +18,15 @@
package org.apache.kylin.common.hll;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.Arrays;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
-import org.apache.commons.compress.utils.IOUtils;
+import org.apache.kylin.common.util.BytesUtil;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
-import org.apache.kylin.common.util.BytesUtil;
-import com.ning.compress.lzf.LZFDecoder;
-import com.ning.compress.lzf.LZFEncoder;
/**
* About compression, test on HLLC data shows
@@ -168,54 +161,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
// ============================================================================
- public static interface Compressor {
-
- byte[] compress(ByteBuffer buf, int offset, int length) throws IOException;
-
- byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException;
- }
-
- static final Compressor GZIP_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- GZIPOutputStream gzout = new GZIPOutputStream(bout);
- gzout.write(buf.array(), offset, length);
- gzout.close();
- return bout.toByteArray();
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayInputStream bin = new ByteArrayInputStream(buf.array(), offset, length);
- GZIPInputStream gzin = new GZIPInputStream(bin);
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- IOUtils.copy(gzin, bout);
- gzin.close();
- bout.close();
- return bout.toByteArray();
- }
- };
-
- static final Compressor LZF_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFEncoder.encode(buf.array(), offset, length);
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFDecoder.decode(buf.array(), offset, length);
- }
- };
-
- public static final int COMPRESSION_THRESHOLD = Integer.MAX_VALUE; // bytes, disable due to slowness
- public static final byte COMPRESSION_FLAG = (byte) 0x02;
- public static final Compressor DEFAULT_COMPRESSOR = GZIP_COMPRESSOR; // LZF lib has a bug at the moment
-
public void writeRegisters(final ByteBuffer out) throws IOException {
- int startPos = out.position();
-
final int indexLen = getRegisterIndexSize();
int size = size();
@@ -240,30 +186,10 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
out.put(registers[i]);
}
}
-
- // do compression if needed
- int len = out.position() - startPos;
- if (len < COMPRESSION_THRESHOLD)
- return;
-
- scheme |= COMPRESSION_FLAG;
- byte[] compressed = DEFAULT_COMPRESSOR.compress(out, startPos + 1, len - 1);
- out.position(startPos);
- out.put(scheme);
- BytesUtil.writeVInt(compressed.length, out);
- out.put(compressed);
}
public void readRegisters(ByteBuffer in) throws IOException {
byte scheme = in.get();
- if ((scheme & COMPRESSION_FLAG) > 0) {
- scheme ^= COMPRESSION_FLAG;
- int compressedLen = BytesUtil.readVInt(in);
- int end = in.position() + compressedLen;
- byte[] decompressed = DEFAULT_COMPRESSOR.decompress(in, in.position(), compressedLen);
- in.position(end);
- in = ByteBuffer.wrap(decompressed);
- }
if (scheme == 0) { // map scheme
clear();