You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/02/12 06:43:35 UTC
[45/50] incubator-kylin git commit: KYLIN-608 rename hllccounter
methond
KYLIN-608 rename hllccounter methond
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ecfc3ccd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ecfc3ccd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ecfc3ccd
Branch: refs/heads/inverted-index
Commit: ecfc3ccd9a2c2a666363d78dced750060b8c6505
Parents: bde1e31
Author: honma <ho...@ebay.com>
Authored: Thu Feb 12 11:03:33 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Thu Feb 12 11:03:33 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 72 ++------------------
.../common/util/HyperLogLogCounterTest.java | 4 +-
.../cardinality/ColumnCardinalityMapper.java | 2 +-
.../cardinality/ColumnCardinalityReducer.java | 4 +-
.../job/tools/ColumnCardinalityMapperTest.java | 4 +-
.../job/tools/ColumnCardinalityReducerTest.java | 2 +-
.../kylin/metadata/measure/HLLCSerializer.java | 4 +-
.../measure/fixedlen/FixedHLLCodec.java | 4 +-
.../query/sqlfunc/HLLDistinctCountAggFunc.java | 4 +-
9 files changed, 18 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index 6ab0611..381dea6 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -169,63 +169,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
// ============================================================================
- public static interface Compressor {
-
- byte[] compress(ByteBuffer buf, int offset, int length) throws IOException;
-
- byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException;
- }
-
- static final Compressor GZIP_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- GZIPOutputStream gzout = new GZIPOutputStream(bout);
- gzout.write(buf.array(), offset, length);
- gzout.close();
- return bout.toByteArray();
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- ByteArrayInputStream bin = new ByteArrayInputStream(buf.array(), offset, length);
- GZIPInputStream gzin = new GZIPInputStream(bin);
- ByteArrayOutputStream bout = new ByteArrayOutputStream();
- IOUtils.copy(gzin, bout);
- gzin.close();
- bout.close();
- return bout.toByteArray();
- }
- };
-
- static final Compressor LZF_COMPRESSOR = new Compressor() {
- @Override
- public byte[] compress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFEncoder.encode(buf.array(), offset, length);
- }
-
- @Override
- public byte[] decompress(ByteBuffer buf, int offset, int length) throws IOException {
- return LZFDecoder.decode(buf.array(), offset, length);
- }
- };
-
- public static final int COMPRESSION_THRESHOLD = Integer.MAX_VALUE; // bytes,
- // disable
- // due to
- // slowness
- public static final byte COMPRESSION_FLAG = (byte) 0x02;
- public static final Compressor DEFAULT_COMPRESSOR = GZIP_COMPRESSOR; // LZF
- // lib
- // has
- // a
- // bug
- // at
- // the
- // moment
-
- public void writeCompactRegisters(final ByteBuffer out) throws IOException {
- int startPos = out.position();
+ public void writeRegisters(final ByteBuffer out) throws IOException {
final int indexLen = getRegisterIndexSize();
int size = size();
@@ -253,7 +197,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
}
}
- public void readCompactRegisters(ByteBuffer in) throws IOException {
+ public void readRegisters(ByteBuffer in) throws IOException {
byte scheme = in.get();
if (scheme == 0) { // map scheme
@@ -271,19 +215,11 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
}
}
- /**
- * For compressed output use writeCompactRegisters
- * @param out
- */
- public void writeRegisters(final ByteBuffer out) {
+ public void writeRegistersArray(final ByteBuffer out) {
out.put(this.registers);
}
- /**
- * For compressed input use readCompactRegisters
- * @param in
- */
- public void readRegisters(ByteBuffer in) {
+ public void readRegistersArray(ByteBuffer in) {
in.get(registers, 0, m);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
index a7d275a..4a438fb 100644
--- a/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/HyperLogLogCounterTest.java
@@ -108,9 +108,9 @@ public class HyperLogLogCounterTest {
private void checkSerialize(HyperLogLogPlusCounter hllc) throws IOException {
long estimate = hllc.getCountEstimate();
buf.clear();
- hllc.writeCompactRegisters(buf);
+ hllc.writeRegisters(buf);
buf.flip();
- hllc.readCompactRegisters(buf);
+ hllc.readRegisters(buf);
Assert.assertEquals(estimate, hllc.getCountEstimate());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
index 329914a..dd7c720 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityMapper.java
@@ -94,7 +94,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, HCatRecord, IntWr
HyperLogLogPlusCounter hllc = hllcMap.get(key);
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
buf.clear();
- hllc.writeCompactRegisters(buf);
+ hllc.writeRegisters(buf);
buf.flip();
context.write(new IntWritable(key), new BytesWritable(buf.array(), buf.limit()));
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
index b4b55d3..a9ac1a3 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/ColumnCardinalityReducer.java
@@ -50,7 +50,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri
for (BytesWritable v : values) {
ByteBuffer buffer = ByteBuffer.wrap(v.getBytes());
HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter();
- hll.readCompactRegisters(buffer);
+ hll.readRegisters(buffer);
getHllc(skey).merge(hll);
hll.clear();
}
@@ -77,7 +77,7 @@ public class ColumnCardinalityReducer extends KylinReducer<IntWritable, BytesWri
HyperLogLogPlusCounter hllc = hllcMap.get(key);
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
buf.clear();
- hllc.writeCompactRegisters(buf);
+ hllc.writeRegisters(buf);
buf.flip();
context.write(new IntWritable(key), new LongWritable(hllc.getCountEstimate()));
// context.write(new Text("ErrorRate_" + key), new
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
index 08c1611..e13289a 100644
--- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityMapperTest.java
@@ -84,7 +84,7 @@ public class ColumnCardinalityMapperTest {
BytesWritable value1 = result.get(0).getSecond();
byte[] bytes = value1.getBytes();
HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
- hllc.readCompactRegisters(ByteBuffer.wrap(bytes));
+ hllc.readRegisters(ByteBuffer.wrap(bytes));
assertTrue(key1 > 0);
assertEquals(8, hllc.getCountEstimate());
}
@@ -117,7 +117,7 @@ public class ColumnCardinalityMapperTest {
BytesWritable value1 = result.get(0).getSecond();
byte[] bytes = value1.getBytes();
HyperLogLogPlusCounter hllc = new HyperLogLogPlusCounter();
- hllc.readCompactRegisters(ByteBuffer.wrap(bytes));
+ hllc.readRegisters(ByteBuffer.wrap(bytes));
System.out.println("ab\177ab".length());
assertTrue(key1 > 0);
assertEquals(1, hllc.getCountEstimate());
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
index 6ad27a8..034c90e 100644
--- a/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
+++ b/job/src/test/java/org/apache/kylin/job/tools/ColumnCardinalityReducerTest.java
@@ -67,7 +67,7 @@ public class ColumnCardinalityReducerTest {
}
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
buf.clear();
- hllc.writeCompactRegisters(buf);
+ hllc.writeRegisters(buf);
buf.flip();
return buf.array();
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
index 20c6c21..e3fcbee 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/HLLCSerializer.java
@@ -38,7 +38,7 @@ public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> {
@Override
public void serialize(HyperLogLogPlusCounter value, ByteBuffer out) {
try {
- value.writeCompactRegisters(out);
+ value.writeRegisters(out);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -47,7 +47,7 @@ public class HLLCSerializer extends MeasureSerializer<HyperLogLogPlusCounter> {
@Override
public HyperLogLogPlusCounter deserialize(ByteBuffer in) {
try {
- current.readCompactRegisters(in);
+ current.readRegisters(in);
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
index d787cbc..c6d4dc9 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/measure/fixedlen/FixedHLLCodec.java
@@ -47,12 +47,12 @@ public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter>
@Override
public HyperLogLogPlusCounter read(byte[] buf, int offset) {
- current.readRegisters(ByteBuffer.wrap(buf, offset, buf.length - offset));
+ current.readRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
return current;
}
@Override
public void write(HyperLogLogPlusCounter v, byte[] buf, int offset) {
- current.writeRegisters(ByteBuffer.wrap(buf, offset, buf.length - offset));
+ current.writeRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ecfc3ccd/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java b/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
index 8d9ace0..356c1e3 100644
--- a/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
+++ b/query/src/main/java/org/apache/kylin/query/sqlfunc/HLLDistinctCountAggFunc.java
@@ -118,12 +118,12 @@ public class HLLDistinctCountAggFunc {
}
@Override
- public void writeCompactRegisters(ByteBuffer out) throws IOException {
+ public void writeRegisters(ByteBuffer out) throws IOException {
throw new UnsupportedOperationException();
}
@Override
- public void readCompactRegisters(ByteBuffer in) throws IOException {
+ public void readRegisters(ByteBuffer in) throws IOException {
throw new UnsupportedOperationException();
}