You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/07 11:30:37 UTC
[1/4] kylin git commit: KYLIN-2338 refactor
BitmapCounter.DataInputByteBuffer
Repository: kylin
Updated Branches:
refs/heads/master 5a18af04c -> 56daf6d59
KYLIN-2338 refactor BitmapCounter.DataInputByteBuffer
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/746e3801
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/746e3801
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/746e3801
Branch: refs/heads/master
Commit: 746e3801faec58f0e10e619f0697ffbe37c08c63
Parents: 5a18af0
Author: kangkaisen <ka...@live.com>
Authored: Mon Dec 26 20:12:30 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jan 7 19:04:52 2017 +0800
----------------------------------------------------------------------
.../kylin/measure/bitmap/BitmapCounter.java | 111 +++----------------
1 file changed, 18 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/746e3801/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index 827390d..a7f277e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -26,7 +26,6 @@ import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Iterator;
-import org.apache.commons.io.IOUtils;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
/**
@@ -107,12 +106,8 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
}
public void readRegisters(ByteBuffer in) throws IOException {
- DataInputByteBuffer input = new DataInputByteBuffer();
- try {
- input.reset(new ByteBuffer[] { in });
- bitmap.deserialize(input);
- } finally {
- IOUtils.closeQuietly(input);
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
+ bitmap.deserialize(is);
}
}
@@ -178,15 +173,11 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
int mark = in.position();
int len;
- DataInputByteBuffer input = new DataInputByteBuffer();
- input.reset(new ByteBuffer[] { in });
MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
- try {
- bitmap.deserialize(input);
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
+ bitmap.deserialize(is);
} catch (IOException e) {
throw new IllegalStateException(e);
- } finally {
- IOUtils.closeQuietly(input);
}
len = in.position() - mark;
@@ -194,93 +185,27 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
return len;
}
- static class DataInputByteBuffer extends DataInputStream {
- private DataInputByteBuffer.Buffer buffers;
+ private class ByteBufferBackedInputStream extends InputStream {
+ private final ByteBuffer buffer;
- public DataInputByteBuffer() {
- this(new DataInputByteBuffer.Buffer());
+ private ByteBufferBackedInputStream(ByteBuffer buf) {
+ buffer = buf;
}
- private DataInputByteBuffer(DataInputByteBuffer.Buffer buffers) {
- super(buffers);
- this.buffers = buffers;
+ @Override
+ public int read() throws IOException {
+ return buffer.hasRemaining() ? (buffer.get() & 0xFF) : -1;
}
- public void reset(ByteBuffer... input) {
- this.buffers.reset(input);
- }
-
- public ByteBuffer[] getData() {
- return this.buffers.getData();
- }
-
- public int getPosition() {
- return this.buffers.getPosition();
- }
-
- public int getLength() {
- return this.buffers.getLength();
- }
-
- private static class Buffer extends InputStream {
- private final byte[] scratch;
- ByteBuffer[] buffers;
- int bidx;
- int pos;
- int length;
-
- private Buffer() {
- this.scratch = new byte[1];
- this.buffers = new ByteBuffer[0];
- }
-
- public int read() {
- return -1 == this.read(this.scratch, 0, 1) ? -1 : this.scratch[0] & 255;
+ @Override
+ public int read(byte[] bytes, int off, int len) throws IOException {
+ if (!buffer.hasRemaining()) {
+ return -1;
}
- public int read(byte[] b, int off, int len) {
- if (this.bidx >= this.buffers.length) {
- return -1;
- } else {
- int cur = 0;
-
- do {
- int rem = Math.min(len, this.buffers[this.bidx].remaining());
- this.buffers[this.bidx].get(b, off, rem);
- cur += rem;
- off += rem;
- len -= rem;
- } while (len > 0 && ++this.bidx < this.buffers.length);
-
- this.pos += cur;
- return cur;
- }
- }
-
- public void reset(ByteBuffer[] buffers) {
- this.bidx = this.pos = this.length = 0;
- this.buffers = buffers;
- ByteBuffer[] arr$ = buffers;
- int len$ = buffers.length;
-
- for (int i$ = 0; i$ < len$; ++i$) {
- ByteBuffer b = arr$[i$];
- this.length += b.remaining();
- }
-
- }
-
- public int getPosition() {
- return this.pos;
- }
-
- public int getLength() {
- return this.length;
- }
-
- public ByteBuffer[] getData() {
- return this.buffers;
- }
+ len = Math.min(len, buffer.remaining());
+ buffer.get(bytes, off, len);
+ return len;
}
}
}
[2/4] kylin git commit: KYLIN-2349 Serialize BitmapCounter with
peekLength
Posted by li...@apache.org.
KYLIN-2349 Serialize BitmapCounter with peekLength
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/aa574462
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/aa574462
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/aa574462
Branch: refs/heads/master
Commit: aa574462aad0620fe10343613dc4c8d761c0d6a0
Parents: 746e380
Author: kangkaisen <ka...@live.com>
Authored: Sat Dec 31 15:41:07 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jan 7 19:08:15 2017 +0800
----------------------------------------------------------------------
.../kylin/measure/bitmap/BitmapCounter.java | 53 ++++++++++++++------
1 file changed, 37 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/aa574462/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index a7f277e..aeb14ba 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -34,6 +34,7 @@ import org.roaringbitmap.buffer.MutableRoaringBitmap;
public class BitmapCounter implements Comparable<BitmapCounter> {
private MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+ private final int VERSION = 2;
public BitmapCounter() {
}
@@ -102,15 +103,51 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
bitmap.serialize(dos);
dos.close();
ByteBuffer bb = ByteBuffer.wrap(bos.toByteArray());
+ out.putInt(VERSION);
+ out.putInt(bos.size() + 4 + 4);
out.put(bb);
}
public void readRegisters(ByteBuffer in) throws IOException {
+ int mark = in.position();
+ int version = in.getInt();
+
+ // keep forward compatibility
+ if (version == VERSION) {
+ @SuppressWarnings("unused")
+ int size = in.getInt();
+ } else {
+ in.position(mark);
+ }
+
try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
bitmap.deserialize(is);
}
}
+ public int peekLength(ByteBuffer in) {
+ int mark = in.position();
+ int len;
+ int version = in.getInt();
+
+ // keep forward compatibility
+ if (version == VERSION) {
+ len = in.getInt() ;
+ } else {
+ in.position(mark);
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
+ MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
+ bitmap.deserialize(is);
+ len = in.position() - mark;
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ in.position(mark);
+ return len;
+ }
+
@Override
public String toString() {
long count = getCount();
@@ -169,22 +206,6 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
return -1;
}
- public int peekLength(ByteBuffer in) {
- int mark = in.position();
- int len;
-
- MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
- try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
- bitmap.deserialize(is);
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
-
- len = in.position() - mark;
- in.position(mark);
- return len;
- }
-
private class ByteBufferBackedInputStream extends InputStream {
private final ByteBuffer buffer;
[4/4] kylin git commit: KYLIN-2353 minor code review
Posted by li...@apache.org.
KYLIN-2353 minor code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56daf6d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56daf6d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56daf6d5
Branch: refs/heads/master
Commit: 56daf6d592ae454b3bca97c4a7a948781485befa
Parents: 731a53a
Author: Yang Li <li...@apache.org>
Authored: Sat Jan 7 19:30:27 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jan 7 19:30:27 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/measure/bitmap/BitmapCounter.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/56daf6d5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index a18ac4a..a632b0d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -51,7 +51,9 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
}
if (buffer != null) {
+ @SuppressWarnings("unused")
int version = buffer.getInt();
+ @SuppressWarnings("unused")
int size = buffer.getInt();
count = buffer.getInt();
@@ -79,7 +81,7 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
public void add(int value) {
getBitmap().add(value);
- count = getBitmap().getCardinality();
+ count = null;
}
public void add(byte[] value) {
@@ -103,12 +105,12 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
public void merge(BitmapCounter another) {
getBitmap().or(another.getBitmap());
- count = getBitmap().getCardinality();
+ count = null;
}
public void intersect(BitmapCounter another) {
getBitmap().and(another.getBitmap());
- count = getBitmap().getCardinality();
+ count = null;
}
public int getCount() {
@@ -148,7 +150,6 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
// keep forward compatibility
if (version == VERSION) {
- @SuppressWarnings("unused")
int size = in.getInt();
count = in.getInt();
in.position(mark);
@@ -258,4 +259,5 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
return len;
}
}
+
}
[3/4] kylin git commit: KYLIN-2353 Serialize BitmapCounter with
distinct count
Posted by li...@apache.org.
KYLIN-2353 Serialize BitmapCounter with distinct count
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/731a53a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/731a53a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/731a53a5
Branch: refs/heads/master
Commit: 731a53a51a32fb1314b4e1282f158e3b7ed819a5
Parents: aa57446
Author: kangkaisen <ka...@live.com>
Authored: Sat Dec 31 20:28:30 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Jan 7 19:08:37 2017 +0800
----------------------------------------------------------------------
.../kylin/measure/bitmap/BitmapCounter.java | 105 ++++++++++++-------
.../bitmap/BitmapDistinctCountAggFunc.java | 2 +-
.../kylin/measure/bitmap/BitmapSerializer.java | 15 +--
3 files changed, 70 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/731a53a5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
index aeb14ba..a18ac4a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapCounter.java
@@ -35,6 +35,8 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
private MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
private final int VERSION = 2;
+ private Integer count;
+ private ByteBuffer buffer;
public BitmapCounter() {
}
@@ -43,18 +45,41 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
merge(another);
}
+ private MutableRoaringBitmap getBitmap() {
+ if (!bitmap.isEmpty()) {
+ return bitmap;
+ }
+
+ if (buffer != null) {
+ int version = buffer.getInt();
+ int size = buffer.getInt();
+ count = buffer.getInt();
+
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(buffer))) {
+ bitmap.deserialize(is);
+ } catch (IOException e) {
+ throw new RuntimeException("deserialize bitmap failed!");
+ }
+
+ buffer = null;
+ }
+
+ return bitmap;
+ }
+
public void clear() {
- bitmap.clear();
+ getBitmap().clear();
}
public BitmapCounter clone() {
BitmapCounter newCounter = new BitmapCounter();
- newCounter.bitmap = bitmap.clone();
+ newCounter.bitmap = getBitmap().clone();
return newCounter;
}
public void add(int value) {
- bitmap.add(value);
+ getBitmap().add(value);
+ count = getBitmap().getCardinality();
}
public void add(byte[] value) {
@@ -77,34 +102,43 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
}
public void merge(BitmapCounter another) {
- this.bitmap.or(another.bitmap);
+ getBitmap().or(another.getBitmap());
+ count = getBitmap().getCardinality();
}
public void intersect(BitmapCounter another) {
- this.bitmap.and(another.bitmap);
+ getBitmap().and(another.getBitmap());
+ count = getBitmap().getCardinality();
}
- public long getCount() {
- return this.bitmap.getCardinality();
+ public int getCount() {
+ if (count != null) {
+ return count;
+ }
+
+ return getBitmap().getCardinality();
}
public int getMemBytes() {
- return this.bitmap.getSizeInBytes();
+ return getBitmap().getSizeInBytes();
}
public Iterator<Integer> iterator() {
- return bitmap.iterator();
+ return getBitmap().iterator();
}
public void writeRegisters(ByteBuffer out) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bos);
+ MutableRoaringBitmap bitmap = getBitmap();
bitmap.runOptimize();
bitmap.serialize(dos);
dos.close();
ByteBuffer bb = ByteBuffer.wrap(bos.toByteArray());
+
out.putInt(VERSION);
- out.putInt(bos.size() + 4 + 4);
+ out.putInt(bos.size() + 4 + 4 + 4);
+ out.putInt(getCount());
out.put(bb);
}
@@ -116,13 +150,30 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
if (version == VERSION) {
@SuppressWarnings("unused")
int size = in.getInt();
+ count = in.getInt();
+ in.position(mark);
+ buffer = cloneBuffer(in, size);
} else {
in.position(mark);
+ try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
+ getBitmap().deserialize(is);
+ }
}
+ }
- try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
- bitmap.deserialize(is);
- }
+ private ByteBuffer cloneBuffer(ByteBuffer src, int size) throws IOException {
+ int mark = src.position();
+ int limit = src.limit();
+
+ src.limit(mark + size);
+ ByteBuffer clone = ByteBuffer.allocate(size);
+ clone.put(src.slice());
+ clone.flip();
+
+ src.position(mark + size);
+ src.limit(limit);
+
+ return clone;
}
public int peekLength(ByteBuffer in) {
@@ -132,7 +183,7 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
// keep forward compatibility
if (version == VERSION) {
- len = in.getInt() ;
+ len = in.getInt();
} else {
in.position(mark);
try (DataInputStream is = new DataInputStream(new ByteBufferBackedInputStream(in))) {
@@ -149,32 +200,10 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
}
@Override
- public String toString() {
- long count = getCount();
- if (count <= 10) {
- return "(" + count + ")" + bitmap.toString();
- } else {
- StringBuilder sb = new StringBuilder();
- sb.append("(").append(count).append("){");
- int values = 0;
- for (Integer v : bitmap) {
- if (values++ < 10) {
- sb.append(v).append(",");
- } else {
- sb.append("...");
- break;
- }
- }
- sb.append("}");
- return sb.toString();
- }
- }
-
- @Override
public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + bitmap.hashCode();
+ result = prime * result + getBitmap().hashCode();
return result;
}
@@ -187,7 +216,7 @@ public class BitmapCounter implements Comparable<BitmapCounter> {
if (getClass() != obj.getClass())
return false;
BitmapCounter other = (BitmapCounter) obj;
- return bitmap.equals(other.bitmap);
+ return getBitmap().equals(other.getBitmap());
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/731a53a5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapDistinctCountAggFunc.java
index d039b6d..3a1a800 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapDistinctCountAggFunc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapDistinctCountAggFunc.java
@@ -35,7 +35,7 @@ public class BitmapDistinctCountAggFunc {
public static BitmapCounter add(BitmapCounter counter, Object v) {
BitmapCounter c = (BitmapCounter) v;
if (counter == null) {
- return new BitmapCounter(c);
+ return c;
} else {
counter.merge(c);
return counter;
http://git-wip-us.apache.org/repos/asf/kylin/blob/731a53a5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
index 089d18c..4890295 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapSerializer.java
@@ -29,8 +29,6 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
*/
public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
- private ThreadLocal<BitmapCounter> current = new ThreadLocal<>();
-
public BitmapSerializer(DataType type) {
}
@@ -43,18 +41,9 @@ public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
}
}
- private BitmapCounter current() {
- BitmapCounter counter = current.get();
- if (counter == null) {
- counter = new BitmapCounter();
- current.set(counter);
- }
- return counter;
- }
-
@Override
public BitmapCounter deserialize(ByteBuffer in) {
- BitmapCounter counter = current();
+ BitmapCounter counter = new BitmapCounter();
try {
counter.readRegisters(in);
} catch (IOException e) {
@@ -65,7 +54,7 @@ public class BitmapSerializer extends DataTypeSerializer<BitmapCounter> {
@Override
public int peekLength(ByteBuffer in) {
- return current().peekLength(in);
+ return new BitmapCounter().peekLength(in);
}
@Override