You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/27 10:27:59 UTC
[02/50] [abbrv] kylin git commit: revert KYLIN-2199 because
KYLIN-2191 is not a major fix
revert KYLIN-2199 because KYLIN-2191 is not a major fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c730b91
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c730b91
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c730b91
Branch: refs/heads/KYLIN-1875
Commit: 1c730b91b3c0847873dbbf74106647b663be1942
Parents: 8132cab
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Nov 22 15:32:31 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Nov 22 15:32:31 2016 +0800
----------------------------------------------------------------------
.../gridtable/DimEncodingPreserveOrderTest.java | 4 +-
.../dimension/DimensionEncodingFactory.java | 4 -
.../apache/kylin/dimension/IntegerDimEnc.java | 3 +-
.../apache/kylin/dimension/IntegerDimEncV2.java | 218 -------------------
.../kylin/dimension/IntegerDimEncTest.java | 16 +-
5 files changed, 11 insertions(+), 234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c730b91/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index 7d6af24..1866079 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -26,7 +26,7 @@ import java.util.List;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.IntegerDimEncV2;
+import org.apache.kylin.dimension.IntegerDimEnc;
import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -98,7 +98,7 @@ public class DimEncodingPreserveOrderTest {
@Test
public void testVLongDimEncPreserveOrder() {
for (int i = 1; i <= successValue.size(); i++) {
- IntegerDimEncV2 enc = new IntegerDimEncV2(i);
+ IntegerDimEnc enc = new IntegerDimEnc(i);
List<ByteArray> encodedValues = Lists.newArrayList();
for (long value : successValue.get(i - 1)) {
encodedValues.add(encode(enc, value));
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c730b91/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 843f4cd..4954ead 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -117,10 +117,6 @@ public abstract class DimensionEncodingFactory {
map.put(Pair.newPair(IntegerDimEnc.ENCODING_NAME, value.getCurrentVersion()), value);
}
{
- IntegerDimEncV2.Factory value = new IntegerDimEncV2.Factory();
- map.put(Pair.newPair(IntegerDimEncV2.ENCODING_NAME, value.getCurrentVersion()), value);
- }
- {
FixedLenHexDimEnc.Factory value = new FixedLenHexDimEnc.Factory();
map.put(Pair.newPair(FixedLenHexDimEnc.ENCODING_NAME, value.getCurrentVersion()), value);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c730b91/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 44d0e73..090dc83 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -36,7 +36,6 @@ import org.slf4j.LoggerFactory;
* -2^(8*N-1) is not supported because the slot is reserved for null values.
* -2^(8*N-1) will be encoded with warn, and its output will be null
*/
-@Deprecated//due to a fatal bug (KYLIN-2191) and the limitation of not able to represent -2^(8N-1)
public class IntegerDimEnc extends DimensionEncoding {
private static final long serialVersionUID = 1L;
@@ -122,7 +121,7 @@ public class IntegerDimEnc extends DimensionEncoding {
//only take useful bytes
integer = integer & MASK[fixedLen];
- boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0;
+ boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0;
if (!positive) {
integer |= (~MASK[fixedLen]);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c730b91/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
deleted file mode 100644
index 6012359..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.dimension;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * replacement for IntDimEnc, the diff is IntegerDimEnc supports negative values
- * for IntegerDimEnc(N), the supported range is (-2^(8*N-1),2^(8*N-1))
- *
- * -2^(8*N-1) is not supported because the slot is reserved for null values.
- * -2^(8*N-1) will be encoded with warn, and its output will be null
- */
-public class IntegerDimEncV2 extends DimensionEncoding {
- private static final long serialVersionUID = 1L;
-
- private static Logger logger = LoggerFactory.getLogger(IntegerDimEncV2.class);
-
- private static final long[] CAP = { 0, 0x7fL, 0x7fffL, 0x7fffffL, 0x7fffffffL, 0x7fffffffffL, 0x7fffffffffffL, 0x7fffffffffffffL, 0x7fffffffffffffffL };
- private static final long[] MASK = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, 0xffffffffffffffffL };
- private static final long[] TAIL = { 0, 0x80L, 0x8000L, 0x800000L, 0x80000000L, 0x8000000000L, 0x800000000000L, 0x80000000000000L, 0x8000000000000000L };
- static {
- for (int i = 1; i < TAIL.length; ++i) {
- long head = ~MASK[i];
- TAIL[i] = head | TAIL[i];
- }
- }
-
- public static final String ENCODING_NAME = "integer";
-
- public static class Factory extends DimensionEncodingFactory {
- @Override
- public String getSupportedEncodingName() {
- return ENCODING_NAME;
- }
-
- @Override
- protected int getCurrentVersion() {
- return 2;
- }
-
- @Override
- public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
- return new IntegerDimEncV2(Integer.parseInt(args[0]));
- }
- };
-
- // ============================================================================
-
- private int fixedLen;
-
- transient private int avoidVerbose = 0;
- transient private int avoidVerbose2 = 0;
-
- //no-arg constructor is required for Externalizable
- public IntegerDimEncV2() {
- }
-
- public IntegerDimEncV2(int len) {
- if (len <= 0 || len >= CAP.length)
- throw new IllegalArgumentException();
-
- this.fixedLen = len;
- }
-
- @Override
- public int getLengthOfEncoding() {
- return fixedLen;
- }
-
- @Override
- public void encode(String valueStr, byte[] output, int outputOffset) {
- if (valueStr == null) {
- Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL);
- return;
- }
-
- long integer = Long.parseLong(valueStr);
- if (integer > CAP[fixedLen] || integer < TAIL[fixedLen]) {
- if (avoidVerbose++ % 10000 == 0) {
- logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose);
- }
- }
-
- if (integer == TAIL[fixedLen]) {
- if (avoidVerbose2++ % 10000 == 0) {
- logger.warn("Value " + valueStr + " does not fit into " + fixedLen + " bytes ");
- }
- }
-
- BytesUtil.writeLong(integer + CAP[fixedLen], output, outputOffset, fixedLen);//apply an offset to preserve binary order, overflow is okay
- }
-
- @Override
- public String decode(byte[] bytes, int offset, int len) {
- if (isNull(bytes, offset, len)) {
- return null;
- }
-
- long integer = BytesUtil.readLong(bytes, offset, len) - CAP[fixedLen];
-
- //only take useful bytes
- integer = integer & MASK[fixedLen];
- boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0;
- if (!positive) {
- integer |= (~MASK[fixedLen]);
- }
-
- return String.valueOf(integer);
- }
-
- @Override
- public DataTypeSerializer<Object> asDataTypeSerializer() {
- return new IntegerSerializer();
- }
-
- public class IntegerSerializer extends DataTypeSerializer<Object> {
- // be thread-safe and avoid repeated obj creation
- private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
-
- private byte[] currentBuf() {
- byte[] buf = current.get();
- if (buf == null) {
- buf = new byte[fixedLen];
- current.set(buf);
- }
- return buf;
- }
-
- @Override
- public void serialize(Object value, ByteBuffer out) {
- byte[] buf = currentBuf();
- String valueStr = value == null ? null : value.toString();
- encode(valueStr, buf, 0);
- out.put(buf);
- }
-
- @Override
- public Object deserialize(ByteBuffer in) {
- byte[] buf = currentBuf();
- in.get(buf);
- return decode(buf, 0, buf.length);
- }
-
- @Override
- public int peekLength(ByteBuffer in) {
- return fixedLen;
- }
-
- @Override
- public int maxLength() {
- return fixedLen;
- }
-
- @Override
- public int getStorageBytesEstimate() {
- return fixedLen;
- }
-
- @Override
- public Object valueOf(String str) {
- return str;
- }
- }
-
- @Override
- public void writeExternal(ObjectOutput out) throws IOException {
- out.writeShort(fixedLen);
- }
-
- @Override
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- fixedLen = in.readShort();
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- IntegerDimEncV2 that = (IntegerDimEncV2) o;
-
- return fixedLen == that.fixedLen;
-
- }
-
- @Override
- public int hashCode() {
- return fixedLen;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c730b91/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
index 0c67bcd..039e056 100644
--- a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
@@ -75,24 +75,24 @@ public class IntegerDimEncTest {
@Test
public void testConstructor() {
try {
- new IntegerDimEncV2(0);
+ new IntegerDimEnc(0);
Assert.fail();
} catch (IllegalArgumentException e) {
// expect
}
try {
- new IntegerDimEncV2(9);
+ new IntegerDimEnc(9);
Assert.fail();
} catch (IllegalArgumentException e) {
// expect
}
- new IntegerDimEncV2(8);
+ new IntegerDimEnc(8);
}
@Test
public void testNull() {
for (int i = 1; i < 9; i++) {
- IntegerDimEncV2 enc = new IntegerDimEncV2(i);
+ IntegerDimEnc enc = new IntegerDimEnc(i);
byte[] buf = new byte[enc.getLengthOfEncoding()];
enc.encode(null, buf, 0);
@@ -112,7 +112,7 @@ public class IntegerDimEncTest {
@Test
public void testEncodeDecode() {
for (int i = 1; i <= successValue.size(); i++) {
- IntegerDimEncV2 enc = new IntegerDimEncV2(i);
+ IntegerDimEnc enc = new IntegerDimEnc(i);
for (long value : successValue.get(i - 1)) {
testEncodeDecode(enc, value);
}
@@ -128,7 +128,7 @@ public class IntegerDimEncTest {
}
}
- private void testEncodeDecode(IntegerDimEncV2 enc, long value) {
+ private void testEncodeDecode(IntegerDimEnc enc, long value) {
String valueStr = "" + value;
byte[] buf = new byte[enc.getLengthOfEncoding()];
enc.encode(valueStr, buf, 0);
@@ -139,7 +139,7 @@ public class IntegerDimEncTest {
@Test
public void testSerDes() {
for (int i = 1; i <= successValue.size(); i++) {
- IntegerDimEncV2 enc = new IntegerDimEncV2(i);
+ IntegerDimEnc enc = new IntegerDimEnc(i);
testSerDes(enc, 127);
for (long value : successValue.get(i - 1)) {
@@ -156,7 +156,7 @@ public class IntegerDimEncTest {
}
}
- private void testSerDes(IntegerDimEncV2 enc, long value) {
+ private void testSerDes(IntegerDimEnc enc, long value) {
DataTypeSerializer<Object> ser = enc.asDataTypeSerializer();
byte[] buf = new byte[enc.getLengthOfEncoding()];
String valueStr = "" + value;