You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/09/27 00:34:39 UTC
[15/50] [abbrv] kylin git commit: deprecate only IntegerDimEnc and
rename SlimLongDimEnc to IntegerDimEnc
deprecate only IntegerDimEnc and rename SlimLongDimEnc to IntegerDimEnc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8babee8d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8babee8d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8babee8d
Branch: refs/heads/1.5.x-CDH5.7
Commit: 8babee8d09f48d8ed60c70f16258402822ea81bf
Parents: f0d2be3
Author: Yang Li <li...@apache.org>
Authored: Wed Sep 21 06:53:10 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Wed Sep 21 06:53:10 2016 +0800
----------------------------------------------------------------------
.../gridtable/DimEncodingPreserveOrderTest.java | 6 +-
.../dimension/DimensionEncodingFactory.java | 2 +-
.../org/apache/kylin/dimension/IntDimEnc.java | 197 +++++++++++++++++
.../apache/kylin/dimension/IntegerDimEnc.java | 39 +++-
.../apache/kylin/dimension/SlimLongDimEnc.java | 219 -------------------
.../apache/kylin/dimension/IntDimEncTest.java | 131 +++++++++++
.../kylin/dimension/IntegerDimEncTest.java | 53 +++--
.../kylin/dimension/SlimLongDimEncTest.java | 151 -------------
8 files changed, 400 insertions(+), 398 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index 0c84ed0..ffd43e5 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.dimension.FixedLenHexDimEnc;
import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
-import org.apache.kylin.dimension.SlimLongDimEnc;
+import org.apache.kylin.dimension.IntegerDimEnc;
import org.junit.Test;
import com.google.common.collect.Lists;
@@ -52,7 +52,7 @@ public class DimEncodingPreserveOrderTest {
@Test
public void testVLongDimEncPreserveOrder() {
- SlimLongDimEnc enc = new SlimLongDimEnc(2);
+ IntegerDimEnc enc = new IntegerDimEnc(2);
List<ByteArray> encodedValues = Lists.newArrayList();
encodedValues.add(encode(enc, -32767L));
encodedValues.add(encode(enc, -10000L));
@@ -68,7 +68,7 @@ public class DimEncodingPreserveOrderTest {
@Test
public void testVLongDimEncPreserveOrder2() {
- SlimLongDimEnc enc = new SlimLongDimEnc(8);
+ IntegerDimEnc enc = new IntegerDimEnc(8);
List<ByteArray> encodedValues = Lists.newArrayList();
encodedValues.add(encode(enc, -Long.MAX_VALUE));
encodedValues.add(encode(enc, -10000L));
http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 739fa25..27bebd7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -74,8 +74,8 @@ public abstract class DimensionEncodingFactory {
// built-in encodings, note dictionary is a special case
map.put(FixedLenDimEnc.ENCODING_NAME, new FixedLenDimEnc.Factory());
+ map.put(IntDimEnc.ENCODING_NAME, new IntDimEnc.Factory());
map.put(IntegerDimEnc.ENCODING_NAME, new IntegerDimEnc.Factory());
- map.put(SlimLongDimEnc.ENCODING_NAME, new SlimLongDimEnc.Factory());
map.put(FixedLenHexDimEnc.ENCODING_NAME, new FixedLenHexDimEnc.Factory());
map.put(DateDimEnc.ENCODING_NAME, new DateDimEnc.Factory());
map.put(TimeDimEnc.ENCODING_NAME, new TimeDimEnc.Factory());
http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
new file mode 100644
index 0000000..88af716
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dimension;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * deprecated use IntegerDimEnc instead
+ * @deprecated
+ */
+public class IntDimEnc extends DimensionEncoding {
+ private static final long serialVersionUID = 1L;
+
+ private static Logger logger = LoggerFactory.getLogger(IntDimEnc.class);
+
+ private static final long[] CAP = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, Long.MAX_VALUE };
+
+ public static final String ENCODING_NAME = "int";
+
+ public static class Factory extends DimensionEncodingFactory {
+ @Override
+ public String getSupportedEncodingName() {
+ return ENCODING_NAME;
+ }
+
+ @Override
+ public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
+ return new IntDimEnc(Integer.parseInt(args[0]));
+ }
+ };
+
+ // ============================================================================
+
+ private int fixedLen;
+
+ transient private int avoidVerbose = 0;
+
+ //no-arg constructor is required for Externalizable
+ public IntDimEnc() {
+ }
+
+ public IntDimEnc(int len) {
+ if (len <= 0 || len >= CAP.length)
+ throw new IllegalArgumentException();
+
+ this.fixedLen = len;
+ }
+
+ @Override
+ public int getLengthOfEncoding() {
+ return fixedLen;
+ }
+
+ @Override
+ public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) {
+ if (value == null) {
+ Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL);
+ return;
+ }
+
+ encode(Bytes.toString(value, 0, valueLen), output, outputOffset);
+ }
+
+ void encode(String valueStr, byte[] output, int outputOffset) {
+ if (valueStr == null) {
+ Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL);
+ return;
+ }
+
+ long integer = Long.parseLong(valueStr);
+ if (integer > CAP[fixedLen]) {
+ if (avoidVerbose++ % 10000 == 0) {
+ logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose);
+ }
+ }
+
+ BytesUtil.writeLong(integer, output, outputOffset, fixedLen);
+ }
+
+ @Override
+ public String decode(byte[] bytes, int offset, int len) {
+ if (isNull(bytes, offset, len)) {
+ return null;
+ }
+
+ long integer = BytesUtil.readLong(bytes, offset, len);
+ return String.valueOf(integer);
+ }
+
+ @Override
+ public DataTypeSerializer<Object> asDataTypeSerializer() {
+ return new IntegerSerializer();
+ }
+
+ public class IntegerSerializer extends DataTypeSerializer<Object> {
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
+
+ private byte[] currentBuf() {
+ byte[] buf = current.get();
+ if (buf == null) {
+ buf = new byte[fixedLen];
+ current.set(buf);
+ }
+ return buf;
+ }
+
+ @Override
+ public void serialize(Object value, ByteBuffer out) {
+ byte[] buf = currentBuf();
+ String valueStr = value == null ? null : value.toString();
+ encode(valueStr, buf, 0);
+ out.put(buf);
+ }
+
+ @Override
+ public Object deserialize(ByteBuffer in) {
+ byte[] buf = currentBuf();
+ in.get(buf);
+ return decode(buf, 0, buf.length);
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return fixedLen;
+ }
+
+ @Override
+ public int maxLength() {
+ return fixedLen;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return fixedLen;
+ }
+
+ @Override
+ public Object valueOf(String str) {
+ return str;
+ }
+ }
+
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeShort(fixedLen);
+ }
+
+ @Override
+ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ fixedLen = in.readShort();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ IntDimEnc that = (IntDimEnc) o;
+
+ return fixedLen == that.fixedLen;
+
+ }
+
+ @Override
+ public int hashCode() {
+ return fixedLen;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index b87e46a..a8cb510 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -31,16 +31,24 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * deprecated use SlimLongDimEnc instead
+ * replacement for IntegerDimEnc, the diff is VLongDimEnc supports negative values
*/
public class IntegerDimEnc extends DimensionEncoding {
private static final long serialVersionUID = 1L;
private static Logger logger = LoggerFactory.getLogger(IntegerDimEnc.class);
- private static final long[] CAP = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, Long.MAX_VALUE };
+ private static final long[] CAP = { 0, 0x7fL, 0x7fffL, 0x7fffffL, 0x7fffffffL, 0x7fffffffffL, 0x7fffffffffffL, 0x7fffffffffffffL, 0x7fffffffffffffffL };
+ private static final long[] MASK = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, 0xffffffffffffffffL };
+ private static final long[] TAIL = { 0, 0x80L, 0x8000L, 0x800000L, 0x80000000L, 0x8000000000L, 0x800000000000L, 0x80000000000000L, 0x8000000000000000L };
+ static {
+ for (int i = 1; i < TAIL.length; ++i) {
+ long head = ~MASK[i];
+ TAIL[i] = head | TAIL[i];
+ }
+ }
- public static final String ENCODING_NAME = "int";
+ public static final String ENCODING_NAME = "integer";
public static class Factory extends DimensionEncodingFactory {
@Override
@@ -59,6 +67,7 @@ public class IntegerDimEnc extends DimensionEncoding {
private int fixedLen;
transient private int avoidVerbose = 0;
+ transient private int avoidVerbose2 = 0;
//no-arg constructor is required for Externalizable
public IntegerDimEnc() {
@@ -93,13 +102,19 @@ public class IntegerDimEnc extends DimensionEncoding {
}
long integer = Long.parseLong(valueStr);
- if (integer > CAP[fixedLen]) {
+ if (integer > CAP[fixedLen] || integer < TAIL[fixedLen]) {
if (avoidVerbose++ % 10000 == 0) {
logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose);
}
}
- BytesUtil.writeLong(integer, output, outputOffset, fixedLen);
+ if (integer == TAIL[fixedLen]) {
+ if (avoidVerbose2++ % 10000 == 0) {
+ logger.warn("Value " + valueStr + " does not fit into " + fixedLen + " bytes ");
+ }
+ }
+
+ BytesUtil.writeLong(integer + CAP[fixedLen], output, outputOffset, fixedLen);//apply an offset to preserve binary order, overflow is okay
}
@Override
@@ -108,7 +123,15 @@ public class IntegerDimEnc extends DimensionEncoding {
return null;
}
- long integer = BytesUtil.readLong(bytes, offset, len);
+ long integer = BytesUtil.readLong(bytes, offset, len) - CAP[fixedLen];
+
+ //only take useful bytes
+ integer = integer & MASK[fixedLen];
+ boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0;
+ if (!positive) {
+ integer |= (~MASK[fixedLen]);
+ }
+
return String.valueOf(integer);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/main/java/org/apache/kylin/dimension/SlimLongDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/SlimLongDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/SlimLongDimEnc.java
deleted file mode 100644
index 4ac871f..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/SlimLongDimEnc.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dimension;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * replacement for IntegerDimEnc, the diff is VLongDimEnc supports negative values
- */
-public class SlimLongDimEnc extends DimensionEncoding {
- private static final long serialVersionUID = 1L;
-
- private static Logger logger = LoggerFactory.getLogger(SlimLongDimEnc.class);
-
- private static final long[] CAP = { 0, 0x7fL, 0x7fffL, 0x7fffffL, 0x7fffffffL, 0x7fffffffffL, 0x7fffffffffffL, 0x7fffffffffffffL, 0x7fffffffffffffffL };
- private static final long[] MASK = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, 0xffffffffffffffffL };
- private static final long[] TAIL = { 0, 0x80L, 0x8000L, 0x800000L, 0x80000000L, 0x8000000000L, 0x800000000000L, 0x80000000000000L, 0x8000000000000000L };
- static {
- for (int i = 1; i < TAIL.length; ++i) {
- long head = ~MASK[i];
- TAIL[i] = head | TAIL[i];
- }
- }
-
- public static final String ENCODING_NAME = "slimlong";
-
- public static class Factory extends DimensionEncodingFactory {
- @Override
- public String getSupportedEncodingName() {
- return ENCODING_NAME;
- }
-
- @Override
- public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
- return new SlimLongDimEnc(Integer.parseInt(args[0]));
- }
- };
-
- // ============================================================================
-
- private int fixedLen;
-
- transient private int avoidVerbose = 0;
- transient private int avoidVerbose2 = 0;
-
- //no-arg constructor is required for Externalizable
- public SlimLongDimEnc() {
- }
-
- public SlimLongDimEnc(int len) {
- if (len <= 0 || len >= CAP.length)
- throw new IllegalArgumentException();
-
- this.fixedLen = len;
- }
-
- @Override
- public int getLengthOfEncoding() {
- return fixedLen;
- }
-
- @Override
- public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) {
- if (value == null) {
- Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL);
- return;
- }
-
- encode(Bytes.toString(value, 0, valueLen), output, outputOffset);
- }
-
- void encode(String valueStr, byte[] output, int outputOffset) {
- if (valueStr == null) {
- Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL);
- return;
- }
-
- long integer = Long.parseLong(valueStr);
- if (integer > CAP[fixedLen] || integer < TAIL[fixedLen]) {
- if (avoidVerbose++ % 10000 == 0) {
- logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose);
- }
- }
-
- if (integer == TAIL[fixedLen]) {
- if (avoidVerbose2++ % 10000 == 0) {
- logger.warn("Value " + valueStr + " does not fit into " + fixedLen + " bytes ");
- }
- }
-
- BytesUtil.writeLong(integer + CAP[fixedLen], output, outputOffset, fixedLen);//apply an offset to preserve binary order, overflow is okay
- }
-
- @Override
- public String decode(byte[] bytes, int offset, int len) {
- if (isNull(bytes, offset, len)) {
- return null;
- }
-
- long integer = BytesUtil.readLong(bytes, offset, len) - CAP[fixedLen];
-
- //only take useful bytes
- integer = integer & MASK[fixedLen];
- boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0;
- if (!positive) {
- integer |= (~MASK[fixedLen]);
- }
-
- return String.valueOf(integer);
- }
-
- @Override
- public DataTypeSerializer<Object> asDataTypeSerializer() {
- return new IntegerSerializer();
- }
-
- public class IntegerSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
-
- private byte[] currentBuf() {
- byte[] buf = current.get();
- if (buf == null) {
- buf = new byte[fixedLen];
- current.set(buf);
- }
- return buf;
- }
-
- @Override
- public void serialize(Object value, ByteBuffer out) {
- byte[] buf = currentBuf();
- String valueStr = value == null ? null : value.toString();
- encode(valueStr, buf, 0);
- out.put(buf);
- }
-
- @Override
- public Object deserialize(ByteBuffer in) {
- byte[] buf = currentBuf();
- in.get(buf);
- return decode(buf, 0, buf.length);
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return fixedLen;
- }
-
- @Override
- public int maxLength() {
- return fixedLen;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return fixedLen;
- }
-
- @Override
- public Object valueOf(String str) {
- return str;
- }
- }
-
- @Override
- public void writeExternal(ObjectOutput out) throws IOException {
- out.writeShort(fixedLen);
- }
-
- @Override
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- fixedLen = in.readShort();
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- SlimLongDimEnc that = (SlimLongDimEnc) o;
-
- return fixedLen == that.fixedLen;
-
- }
-
- @Override
- public int hashCode() {
- return fixedLen;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java
new file mode 100644
index 0000000..280a242
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dimension;
+
+import java.nio.ByteBuffer;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Deprecated. use VLongDimEnc instead
+ * @deprecated
+ */
+public class IntDimEncTest {
+
+ @Test
+ public void testConstructor() {
+ try {
+ new IntDimEnc(0);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ // expect
+ }
+ try {
+ new IntDimEnc(9);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ // expect
+ }
+ new IntDimEnc(8);
+ }
+
+ @Test
+ public void testNull() {
+ for (int i = 1; i < 9; i++) {
+ IntDimEnc enc = new IntDimEnc(i);
+
+ byte[] buf = new byte[enc.getLengthOfEncoding()];
+ enc.encode(null, 0, buf, 0);
+ Assert.assertTrue(DimensionEncoding.isNull(buf, 0, buf.length));
+ String decode = enc.decode(buf, 0, buf.length);
+ Assert.assertEquals(null, decode);
+
+ buf = new byte[enc.getLengthOfEncoding()];
+ DataTypeSerializer<Object> ser = enc.asDataTypeSerializer();
+ ser.serialize(null, ByteBuffer.wrap(buf));
+ Assert.assertTrue(DimensionEncoding.isNull(buf, 0, buf.length));
+ decode = (String) ser.deserialize(ByteBuffer.wrap(buf));
+ Assert.assertEquals(null, decode);
+ }
+ }
+
+ @Test
+ public void testEncodeDecode() {
+ IntDimEnc enc = new IntDimEnc(2);
+ testEncodeDecode(enc, 0);
+ testEncodeDecode(enc, 100);
+ testEncodeDecode(enc, 10000);
+ testEncodeDecode(enc, 65534);
+ try {
+ testEncodeDecode(enc, 65535);
+ Assert.fail();
+ } catch (Throwable e) {
+ Assert.assertEquals("expected:<65535> but was:<null>", e.getMessage());
+ }
+ try {
+ testEncodeDecode(enc, 65536);
+ Assert.fail();
+ } catch (Throwable e) {
+ Assert.assertEquals("expected:<[65536]> but was:<[0]>", e.getMessage());
+ }
+ }
+
+ private void testEncodeDecode(IntDimEnc enc, long value) {
+ byte[] buf = new byte[enc.getLengthOfEncoding()];
+ String valueStr = "" + value;
+ byte[] bytes = Bytes.toBytes(valueStr);
+ enc.encode(bytes, bytes.length, buf, 0);
+ String decode = enc.decode(buf, 0, buf.length);
+ Assert.assertEquals(valueStr, decode);
+ }
+
+ @Test
+ public void testSerDes() {
+ IntDimEnc enc = new IntDimEnc(2);
+ testSerDes(enc, 0);
+ testSerDes(enc, 100);
+ testSerDes(enc, 10000);
+ testSerDes(enc, 65534);
+ try {
+ testSerDes(enc, 65535);
+ Assert.fail();
+ } catch (Throwable e) {
+ Assert.assertEquals("expected:<65535> but was:<null>", e.getMessage());
+ }
+ try {
+ testSerDes(enc, 65536);
+ Assert.fail();
+ } catch (Throwable e) {
+ Assert.assertEquals("expected:<[65536]> but was:<[0]>", e.getMessage());
+ }
+ }
+
+ private void testSerDes(IntDimEnc enc, long value) {
+ DataTypeSerializer<Object> ser = enc.asDataTypeSerializer();
+ byte[] buf = new byte[enc.getLengthOfEncoding()];
+ String valueStr = "" + value;
+ ser.serialize(valueStr, ByteBuffer.wrap(buf));
+ String decode = (String) ser.deserialize(ByteBuffer.wrap(buf));
+ Assert.assertEquals(valueStr, decode);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
index fccd8b9..1bdb1d0 100644
--- a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -25,9 +25,6 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.junit.Assert;
import org.junit.Test;
-/**
- * Deprecated. use VLongDimEnc instead
- */
public class IntegerDimEncTest {
@Test
@@ -73,24 +70,45 @@ public class IntegerDimEncTest {
testEncodeDecode(enc, 0);
testEncodeDecode(enc, 100);
testEncodeDecode(enc, 10000);
- testEncodeDecode(enc, 65534);
+ testEncodeDecode(enc, 32767);
+ testEncodeDecode(enc, -100);
+ testEncodeDecode(enc, -10000);
+ testEncodeDecode(enc, -32767);
try {
- testEncodeDecode(enc, 65535);
+ testEncodeDecode(enc, 32768);
Assert.fail();
} catch (Throwable e) {
- Assert.assertEquals("expected:<65535> but was:<null>", e.getMessage());
+ Assert.assertEquals("expected:<32768> but was:<null>", e.getMessage());
}
try {
- testEncodeDecode(enc, 65536);
+ testEncodeDecode(enc, -32768);
Assert.fail();
} catch (Throwable e) {
- Assert.assertEquals("expected:<[65536]> but was:<[0]>", e.getMessage());
+ Assert.assertEquals("expected:<-32768> but was:<null>", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testEncodeDecode2() {
+ IntegerDimEnc enc = new IntegerDimEnc(8);
+ testEncodeDecode(enc, 0);
+ testEncodeDecode(enc, 100);
+ testEncodeDecode(enc, 10000);
+ testEncodeDecode(enc, Long.MAX_VALUE);
+ testEncodeDecode(enc, -100);
+ testEncodeDecode(enc, -10000);
+ testEncodeDecode(enc, -Long.MAX_VALUE);
+ try {
+ testEncodeDecode(enc, Long.MIN_VALUE);
+ Assert.fail();
+ } catch (Throwable e) {
+ Assert.assertEquals("expected:<-9223372036854775808> but was:<null>", e.getMessage());
}
}
private void testEncodeDecode(IntegerDimEnc enc, long value) {
- byte[] buf = new byte[enc.getLengthOfEncoding()];
String valueStr = "" + value;
+ byte[] buf = new byte[enc.getLengthOfEncoding()];
byte[] bytes = Bytes.toBytes(valueStr);
enc.encode(bytes, bytes.length, buf, 0);
String decode = enc.decode(buf, 0, buf.length);
@@ -103,18 +121,21 @@ public class IntegerDimEncTest {
testSerDes(enc, 0);
testSerDes(enc, 100);
testSerDes(enc, 10000);
- testSerDes(enc, 65534);
+ testSerDes(enc, 32767);
+ testSerDes(enc, -100);
+ testSerDes(enc, -10000);
+ testSerDes(enc, -32767);
try {
- testSerDes(enc, 65535);
+ testSerDes(enc, 32768);
Assert.fail();
} catch (Throwable e) {
- Assert.assertEquals("expected:<65535> but was:<null>", e.getMessage());
+ Assert.assertEquals("expected:<32768> but was:<null>", e.getMessage());
}
try {
- testSerDes(enc, 65536);
+ testSerDes(enc, -32768);
Assert.fail();
} catch (Throwable e) {
- Assert.assertEquals("expected:<[65536]> but was:<[0]>", e.getMessage());
+ Assert.assertEquals("expected:<-32768> but was:<null>", e.getMessage());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8babee8d/core-metadata/src/test/java/org/apache/kylin/dimension/SlimLongDimEncTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/SlimLongDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/SlimLongDimEncTest.java
deleted file mode 100644
index a1e9516..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/dimension/SlimLongDimEncTest.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dimension;
-
-import java.nio.ByteBuffer;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class SlimLongDimEncTest {
-
- @Test
- public void testConstructor() {
- try {
- new SlimLongDimEnc(0);
- Assert.fail();
- } catch (IllegalArgumentException e) {
- // expect
- }
- try {
- new SlimLongDimEnc(9);
- Assert.fail();
- } catch (IllegalArgumentException e) {
- // expect
- }
- new SlimLongDimEnc(8);
- }
-
- @Test
- public void testNull() {
- for (int i = 1; i < 9; i++) {
- SlimLongDimEnc enc = new SlimLongDimEnc(i);
-
- byte[] buf = new byte[enc.getLengthOfEncoding()];
- enc.encode(null, 0, buf, 0);
- Assert.assertTrue(DimensionEncoding.isNull(buf, 0, buf.length));
- String decode = enc.decode(buf, 0, buf.length);
- Assert.assertEquals(null, decode);
-
- buf = new byte[enc.getLengthOfEncoding()];
- DataTypeSerializer<Object> ser = enc.asDataTypeSerializer();
- ser.serialize(null, ByteBuffer.wrap(buf));
- Assert.assertTrue(DimensionEncoding.isNull(buf, 0, buf.length));
- decode = (String) ser.deserialize(ByteBuffer.wrap(buf));
- Assert.assertEquals(null, decode);
- }
- }
-
- @Test
- public void testEncodeDecode() {
- SlimLongDimEnc enc = new SlimLongDimEnc(2);
- testEncodeDecode(enc, 0);
- testEncodeDecode(enc, 100);
- testEncodeDecode(enc, 10000);
- testEncodeDecode(enc, 32767);
- testEncodeDecode(enc, -100);
- testEncodeDecode(enc, -10000);
- testEncodeDecode(enc, -32767);
- try {
- testEncodeDecode(enc, 32768);
- Assert.fail();
- } catch (Throwable e) {
- Assert.assertEquals("expected:<32768> but was:<null>", e.getMessage());
- }
- try {
- testEncodeDecode(enc, -32768);
- Assert.fail();
- } catch (Throwable e) {
- Assert.assertEquals("expected:<-32768> but was:<null>", e.getMessage());
- }
- }
-
- @Test
- public void testEncodeDecode2() {
- SlimLongDimEnc enc = new SlimLongDimEnc(8);
- testEncodeDecode(enc, 0);
- testEncodeDecode(enc, 100);
- testEncodeDecode(enc, 10000);
- testEncodeDecode(enc, Long.MAX_VALUE);
- testEncodeDecode(enc, -100);
- testEncodeDecode(enc, -10000);
- testEncodeDecode(enc, -Long.MAX_VALUE);
- try {
- testEncodeDecode(enc, Long.MIN_VALUE);
- Assert.fail();
- } catch (Throwable e) {
- Assert.assertEquals("expected:<-9223372036854775808> but was:<null>", e.getMessage());
- }
- }
-
- private void testEncodeDecode(SlimLongDimEnc enc, long value) {
- String valueStr = "" + value;
- byte[] buf = new byte[enc.getLengthOfEncoding()];
- byte[] bytes = Bytes.toBytes(valueStr);
- enc.encode(bytes, bytes.length, buf, 0);
- String decode = enc.decode(buf, 0, buf.length);
- Assert.assertEquals(valueStr, decode);
- }
-
- @Test
- public void testSerDes() {
- SlimLongDimEnc enc = new SlimLongDimEnc(2);
- testSerDes(enc, 0);
- testSerDes(enc, 100);
- testSerDes(enc, 10000);
- testSerDes(enc, 32767);
- testSerDes(enc, -100);
- testSerDes(enc, -10000);
- testSerDes(enc, -32767);
- try {
- testSerDes(enc, 32768);
- Assert.fail();
- } catch (Throwable e) {
- Assert.assertEquals("expected:<32768> but was:<null>", e.getMessage());
- }
- try {
- testSerDes(enc, -32768);
- Assert.fail();
- } catch (Throwable e) {
- Assert.assertEquals("expected:<-32768> but was:<null>", e.getMessage());
- }
- }
-
- private void testSerDes(SlimLongDimEnc enc, long value) {
- DataTypeSerializer<Object> ser = enc.asDataTypeSerializer();
- byte[] buf = new byte[enc.getLengthOfEncoding()];
- String valueStr = "" + value;
- ser.serialize(valueStr, ByteBuffer.wrap(buf));
- String decode = (String) ser.deserialize(ByteBuffer.wrap(buf));
- Assert.assertEquals(valueStr, decode);
- }
-
-}