You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/11/17 03:03:29 UTC
[1/2] kylin git commit: KYLIN-2199 refined KYLIN-2191 due to
KYLIN-2198
Repository: kylin
Updated Branches:
refs/heads/master 21a1c5c09 -> 4fcaa8ac3
KYLIN-2199 refined KYLIN-2191 due to KYLIN-2198
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4fcaa8ac
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4fcaa8ac
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4fcaa8ac
Branch: refs/heads/master
Commit: 4fcaa8ac3fdeb7709ba53b531393ab88a71ecdd5
Parents: caf7040
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Nov 16 14:48:08 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Nov 17 10:56:36 2016 +0800
----------------------------------------------------------------------
.../gridtable/DimEncodingPreserveOrderTest.java | 4 +-
.../apache/kylin/dimension/IntegerDimEnc.java | 8 +-
.../apache/kylin/dimension/IntegerDimEncV2.java | 228 +++++++++++++++++++
.../apache/kylin/dimension/IntDimEncTest.java | 2 +-
.../kylin/dimension/IntegerDimEncTest.java | 18 +-
5 files changed, 247 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fcaa8ac/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
index 87d37be..d572e56 100644
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/gridtable/DimEncodingPreserveOrderTest.java
@@ -27,7 +27,7 @@ import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.dimension.FixedLenHexDimEnc;
-import org.apache.kylin.dimension.IntegerDimEnc;
+import org.apache.kylin.dimension.IntegerDimEncV2;
import org.apache.kylin.dimension.OneMoreByteVLongDimEnc;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -99,7 +99,7 @@ public class DimEncodingPreserveOrderTest {
@Test
public void testVLongDimEncPreserveOrder() {
for (int i = 1; i <= successValue.size(); i++) {
- IntegerDimEnc enc = new IntegerDimEnc(i);
+ IntegerDimEncV2 enc = new IntegerDimEncV2(i);
List<ByteArray> encodedValues = Lists.newArrayList();
for (long value : successValue.get(i - 1)) {
encodedValues.add(encode(enc, value));
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fcaa8ac/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 460ca65..c417e37 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -32,8 +32,12 @@ import org.slf4j.LoggerFactory;
/**
* replacement for IntDimEnc, the diff is IntegerDimEnc supports negative values
- * for IntegerDimEnc(N), the supported range is [-2^(8N-1),2^(8N-1)]
+ * for IntegerDimEnc(N), the supported range is (-2^(8*N-1),2^(8*N-1))
+ *
+ * -2^(8*N-1) is not supported because the slot is reserved for null values.
+ * -2^(8*N-1) will be encoded with warn, and its output will be null
*/
+@Deprecated//due to a fatal bug (KYLIN-2191) and the limitation of not able to represent -2^(8N-1)
public class IntegerDimEnc extends DimensionEncoding {
private static final long serialVersionUID = 1L;
@@ -128,7 +132,7 @@ public class IntegerDimEnc extends DimensionEncoding {
//only take useful bytes
integer = integer & MASK[fixedLen];
- boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0;
+ boolean positive = (integer & ((0x80) << ((fixedLen - 1) << 3))) == 0;
if (!positive) {
integer |= (~MASK[fixedLen]);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fcaa8ac/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
new file mode 100644
index 0000000..1a54664
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEncV2.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.dimension;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * replacement for IntDimEnc, the diff is IntegerDimEnc supports negative values
+ * for IntegerDimEnc(N), the supported range is (-2^(8*N-1),2^(8*N-1))
+ *
+ * -2^(8*N-1) is not supported because the slot is reserved for null values.
+ * -2^(8*N-1) will be encoded with warn, and its output will be null
+ */
+public class IntegerDimEncV2 extends DimensionEncoding {
+ private static final long serialVersionUID = 1L;
+
+ private static Logger logger = LoggerFactory.getLogger(IntegerDimEncV2.class);
+
+ private static final long[] CAP = { 0, 0x7fL, 0x7fffL, 0x7fffffL, 0x7fffffffL, 0x7fffffffffL, 0x7fffffffffffL, 0x7fffffffffffffL, 0x7fffffffffffffffL };
+ private static final long[] MASK = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, 0xffffffffffffffffL };
+ private static final long[] TAIL = { 0, 0x80L, 0x8000L, 0x800000L, 0x80000000L, 0x8000000000L, 0x800000000000L, 0x80000000000000L, 0x8000000000000000L };
+ static {
+ for (int i = 1; i < TAIL.length; ++i) {
+ long head = ~MASK[i];
+ TAIL[i] = head | TAIL[i];
+ }
+ }
+
+ public static final String ENCODING_NAME = "integer";
+
+ public static class Factory extends DimensionEncodingFactory {
+ @Override
+ public String getSupportedEncodingName() {
+ return ENCODING_NAME;
+ }
+
+ @Override
+ protected int getCurrentVersion() {
+ return 2;
+ }
+
+ @Override
+ public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
+ return new IntegerDimEncV2(Integer.parseInt(args[0]));
+ }
+ };
+
+ // ============================================================================
+
+ private int fixedLen;
+
+ transient private int avoidVerbose = 0;
+ transient private int avoidVerbose2 = 0;
+
+ //no-arg constructor is required for Externalizable
+ public IntegerDimEncV2() {
+ }
+
+ public IntegerDimEncV2(int len) {
+ if (len <= 0 || len >= CAP.length)
+ throw new IllegalArgumentException();
+
+ this.fixedLen = len;
+ }
+
+ @Override
+ public int getLengthOfEncoding() {
+ return fixedLen;
+ }
+
+ @Override
+ public void encode(byte[] value, int valueLen, byte[] output, int outputOffset) {
+ if (value == null) {
+ Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL);
+ return;
+ }
+
+ encode(Bytes.toString(value, 0, valueLen), output, outputOffset);
+ }
+
+ void encode(String valueStr, byte[] output, int outputOffset) {
+ if (valueStr == null) {
+ Arrays.fill(output, outputOffset, outputOffset + fixedLen, NULL);
+ return;
+ }
+
+ long integer = Long.parseLong(valueStr);
+ if (integer > CAP[fixedLen] || integer < TAIL[fixedLen]) {
+ if (avoidVerbose++ % 10000 == 0) {
+ logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose);
+ }
+ }
+
+ if (integer == TAIL[fixedLen]) {
+ if (avoidVerbose2++ % 10000 == 0) {
+ logger.warn("Value " + valueStr + " does not fit into " + fixedLen + " bytes ");
+ }
+ }
+
+ BytesUtil.writeLong(integer + CAP[fixedLen], output, outputOffset, fixedLen);//apply an offset to preserve binary order, overflow is okay
+ }
+
+ @Override
+ public String decode(byte[] bytes, int offset, int len) {
+ if (isNull(bytes, offset, len)) {
+ return null;
+ }
+
+ long integer = BytesUtil.readLong(bytes, offset, len) - CAP[fixedLen];
+
+ //only take useful bytes
+ integer = integer & MASK[fixedLen];
+ boolean positive = (integer & ((0x80L) << ((fixedLen - 1) << 3))) == 0;
+ if (!positive) {
+ integer |= (~MASK[fixedLen]);
+ }
+
+ return String.valueOf(integer);
+ }
+
+ @Override
+ public DataTypeSerializer<Object> asDataTypeSerializer() {
+ return new IntegerSerializer();
+ }
+
+ public class IntegerSerializer extends DataTypeSerializer<Object> {
+ // be thread-safe and avoid repeated obj creation
+ private ThreadLocal<byte[]> current = new ThreadLocal<byte[]>();
+
+ private byte[] currentBuf() {
+ byte[] buf = current.get();
+ if (buf == null) {
+ buf = new byte[fixedLen];
+ current.set(buf);
+ }
+ return buf;
+ }
+
+ @Override
+ public void serialize(Object value, ByteBuffer out) {
+ byte[] buf = currentBuf();
+ String valueStr = value == null ? null : value.toString();
+ encode(valueStr, buf, 0);
+ out.put(buf);
+ }
+
+ @Override
+ public Object deserialize(ByteBuffer in) {
+ byte[] buf = currentBuf();
+ in.get(buf);
+ return decode(buf, 0, buf.length);
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return fixedLen;
+ }
+
+ @Override
+ public int maxLength() {
+ return fixedLen;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return fixedLen;
+ }
+
+ @Override
+ public Object valueOf(String str) {
+ return str;
+ }
+ }
+
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeShort(fixedLen);
+ }
+
+ @Override
+ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ fixedLen = in.readShort();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ IntegerDimEncV2 that = (IntegerDimEncV2) o;
+
+ return fixedLen == that.fixedLen;
+
+ }
+
+ @Override
+ public int hashCode() {
+ return fixedLen;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fcaa8ac/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java
index 280a242..d228dd5 100644
--- a/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java
@@ -26,7 +26,7 @@ import org.junit.Assert;
import org.junit.Test;
/**
- * Deprecated. use VLongDimEnc instead
+ * Deprecated. use integer encoding instead
* @deprecated
*/
public class IntDimEncTest {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4fcaa8ac/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
index a9b2511..9924053 100644
--- a/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/dimension/IntegerDimEncTest.java
@@ -76,24 +76,24 @@ public class IntegerDimEncTest {
@Test
public void testConstructor() {
try {
- new IntegerDimEnc(0);
+ new IntegerDimEncV2(0);
Assert.fail();
} catch (IllegalArgumentException e) {
// expect
}
try {
- new IntegerDimEnc(9);
+ new IntegerDimEncV2(9);
Assert.fail();
} catch (IllegalArgumentException e) {
// expect
}
- new IntegerDimEnc(8);
+ new IntegerDimEncV2(8);
}
@Test
public void testNull() {
for (int i = 1; i < 9; i++) {
- IntegerDimEnc enc = new IntegerDimEnc(i);
+ IntegerDimEncV2 enc = new IntegerDimEncV2(i);
byte[] buf = new byte[enc.getLengthOfEncoding()];
enc.encode(null, 0, buf, 0);
@@ -113,7 +113,7 @@ public class IntegerDimEncTest {
@Test
public void testEncodeDecode() {
for (int i = 1; i <= successValue.size(); i++) {
- IntegerDimEnc enc = new IntegerDimEnc(i);
+ IntegerDimEncV2 enc = new IntegerDimEncV2(i);
for (long value : successValue.get(i - 1)) {
testEncodeDecode(enc, value);
}
@@ -129,7 +129,7 @@ public class IntegerDimEncTest {
}
}
- private void testEncodeDecode(IntegerDimEnc enc, long value) {
+ private void testEncodeDecode(IntegerDimEncV2 enc, long value) {
String valueStr = "" + value;
byte[] buf = new byte[enc.getLengthOfEncoding()];
byte[] bytes = Bytes.toBytes(valueStr);
@@ -141,7 +141,9 @@ public class IntegerDimEncTest {
@Test
public void testSerDes() {
for (int i = 1; i <= successValue.size(); i++) {
- IntegerDimEnc enc = new IntegerDimEnc(i);
+ IntegerDimEncV2 enc = new IntegerDimEncV2(i);
+
+ testSerDes(enc, 127);
for (long value : successValue.get(i - 1)) {
testSerDes(enc, value);
}
@@ -156,7 +158,7 @@ public class IntegerDimEncTest {
}
}
- private void testSerDes(IntegerDimEnc enc, long value) {
+ private void testSerDes(IntegerDimEncV2 enc, long value) {
DataTypeSerializer<Object> ser = enc.asDataTypeSerializer();
byte[] buf = new byte[enc.getLengthOfEncoding()];
String valueStr = "" + value;
[2/2] kylin git commit: KYLIN-2198 Add a framework to allow major
changes in DimensionEncoding
Posted by ma...@apache.org.
KYLIN-2198 Add a framework to allow major changes in DimensionEncoding
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/caf7040f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/caf7040f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/caf7040f
Branch: refs/heads/master
Commit: caf7040fc745ccca2525300979fce7785d6f0969
Parents: 21a1c5c
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Nov 16 14:47:53 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Nov 17 10:56:36 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/JacksonBean.java | 55 ++++++++++
.../apache/kylin/common/util/JacksonTest.java | 39 +++++++
.../org/apache/kylin/cube/kv/CubeDimEncMap.java | 2 +-
.../apache/kylin/cube/model/RowKeyColDesc.java | 13 ++-
.../dimension/DimensionEncodingFactory.java | 102 ++++++++++++++-----
.../kylin/measure/topn/TopNMeasureType.java | 12 ++-
.../kylin/rest/controller/CubeController.java | 13 ++-
7 files changed, 202 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/caf7040f/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
new file mode 100644
index 0000000..42357f2
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonBean.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class JacksonBean {
+
+ @JsonProperty("a")
+ private String a;
+ @JsonProperty("b")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private int b;
+
+ public String getA() {
+ return a;
+ }
+
+ public void setA(String a) {
+ this.a = a;
+ }
+
+ public int getB() {
+ return b;
+ }
+
+ public void setB(int b) {
+ this.b = b;
+ }
+
+ @Override
+ public String toString() {
+ return "JacksonBean{" + "a='" + a + '\'' + ", b=" + b + '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/caf7040f/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
new file mode 100644
index 0000000..81be7eb
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/JacksonTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.util;
+
+import java.io.IOException;
+
+import org.junit.Test;
+
+public class JacksonTest {
+ @Test
+ public void foo() throws IOException {
+ JacksonBean bean = new JacksonBean();
+ bean.setA("valuea");
+
+ String s = JsonUtil.writeValueAsString(bean);
+ System.out.println(s);
+
+ JacksonBean desBean = (JacksonBean) JsonUtil.readValue("{\"a\":\"valuea\"}", JacksonBean.class);
+ String x2 = JsonUtil.writeValueAsString(desBean);
+ System.out.println(desBean);
+ System.out.println(x2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/caf7040f/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
index f588986..a4d2d6f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
@@ -72,7 +72,7 @@ public class CubeDimEncMap implements IDimensionEncodingMap {
}
} else {
// normal case
- result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs());
+ result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion());
}
encMap.put(col, result);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/caf7040f/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 6192d76..cc264d5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -46,6 +46,9 @@ public class RowKeyColDesc {
private String column;
@JsonProperty("encoding")
private String encoding;
+ @JsonProperty("encoding_version")
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ private int encodingVersion = 1;
@JsonProperty("isShardBy")
private boolean isShardBy;//usually it is ultra high cardinality column, shard by such column can reduce the agg cache for each shard
@JsonProperty("index")
@@ -69,7 +72,7 @@ public class RowKeyColDesc {
encodingName = (String) encodingConf[0];
encodingArgs = (String[]) encodingConf[1];
- if (!DimensionEncodingFactory.isVaildEncoding(this.encodingName))
+ if (!DimensionEncodingFactory.isValidEncoding(this.encodingName))
throw new IllegalArgumentException("Not supported row key col encoding: '" + this.encoding + "'");
// convert date/time dictionary on date/time column to DimensionEncoding implicitly
@@ -143,6 +146,14 @@ public class RowKeyColDesc {
this.index = index;
}
+ public int getEncodingVersion() {
+ return encodingVersion;
+ }
+
+ public void setEncodingVersion(int encodingVersion) {
+ this.encodingVersion = encodingVersion;
+ }
+
@Override
public String toString() {
return Objects.toStringHelper(this).add("column", column).add("encoding", encoding).toString();
http://git-wip-us.apache.org/repos/asf/kylin/blob/caf7040f/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index aba0c26..37d1014 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -18,30 +18,47 @@
package org.apache.kylin.dimension;
+import java.util.Arrays;
import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
+
+import javax.annotation.Nullable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
public abstract class DimensionEncodingFactory {
private static final Logger logger = LoggerFactory.getLogger(DimensionEncodingFactory.class);
- private static Map<String, DimensionEncodingFactory> factoryMap;
+ private static Map<Pair<String, Integer>, DimensionEncodingFactory> factoryMap;
+
+ /**
+ * If a bug found in a DimEnc will cause different cube outputs,
+ * we'll have to increase the version number of DimEnc, in order
+ * to distinguish current version with prior version.
+ * <p>
+ * The default version applys to all existing legacy DimEncs
+ */
+ protected int getCurrentVersion() {
+ return 1;
+ }
- /** Create a DimensionEncoding instance, with inputs corresponding to RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs. */
- public static DimensionEncoding create(String encodingName, String[] args) {
+ /**
+ * Create a DimensionEncoding instance, with inputs corresponding to RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs.
+ */
+ public static DimensionEncoding create(String encodingName, String[] args, int version) {
+ logger.debug("Encoding Name : {}, args : {}, version {}", encodingName, Arrays.toString(args), version);
if (factoryMap == null)
initFactoryMap();
- DimensionEncodingFactory factory = factoryMap.get(encodingName);
+ DimensionEncodingFactory factory = factoryMap.get(Pair.newPair(encodingName, version));
if (factory == null) {
throw new IllegalArgumentException("Unknown dimension encoding name " + encodingName //
+ " (note '" + DictionaryDimEnc.ENCODING_NAME + "' is not handled by factory)");
@@ -50,43 +67,76 @@ public abstract class DimensionEncodingFactory {
return factory.createDimensionEncoding(encodingName, args);
}
- public static Set<String> getValidEncodings() {
+ public static Map<String, Integer> getValidEncodings() {
if (factoryMap == null)
initFactoryMap();
- TreeSet<String> result = Sets.newTreeSet();
- result.addAll(factoryMap.keySet());
- result.add(DictionaryDimEnc.ENCODING_NAME);
+ Map<String, Integer> result = Maps.newHashMap();
+ for (Pair<String, Integer> p : factoryMap.keySet()) {
+ result.put(p.getFirst(), p.getSecond());
+ }
+ result.put(DictionaryDimEnc.ENCODING_NAME, 1);
return result;
}
- public static boolean isVaildEncoding(String encodingName) {
+ public static boolean isValidEncoding(final String encodingName) {
if (factoryMap == null)
initFactoryMap();
// note dictionary is a special case
- return DictionaryDimEnc.ENCODING_NAME.equals(encodingName) || factoryMap.containsKey(encodingName);
+ return DictionaryDimEnc.ENCODING_NAME.equals(encodingName) || //
+ Iterables.any(factoryMap.keySet(), new Predicate<Pair<String, Integer>>() {
+ @Override
+ public boolean apply(@Nullable Pair<String, Integer> input) {
+ return input.getFirst().equals(encodingName);
+ }
+ });
}
private synchronized static void initFactoryMap() {
if (factoryMap == null) {
- Map<String, DimensionEncodingFactory> map = Maps.newConcurrentMap();
+ Map<Pair<String, Integer>, DimensionEncodingFactory> map = Maps.newConcurrentMap();
// built-in encodings, note dictionary is a special case
- map.put(FixedLenDimEnc.ENCODING_NAME, new FixedLenDimEnc.Factory());
- map.put(IntDimEnc.ENCODING_NAME, new IntDimEnc.Factory());
- map.put(IntegerDimEnc.ENCODING_NAME, new IntegerDimEnc.Factory());
- map.put(FixedLenHexDimEnc.ENCODING_NAME, new FixedLenHexDimEnc.Factory());
- map.put(DateDimEnc.ENCODING_NAME, new DateDimEnc.Factory());
- map.put(TimeDimEnc.ENCODING_NAME, new TimeDimEnc.Factory());
- map.put(BooleanDimEnc.ENCODING_NAME, new BooleanDimEnc.Factory());
+ {
+ FixedLenDimEnc.Factory value = new FixedLenDimEnc.Factory();
+ map.put(Pair.newPair(FixedLenDimEnc.ENCODING_NAME, value.getCurrentVersion()), value);
+ }
+ {
+ IntDimEnc.Factory value = new IntDimEnc.Factory();
+ map.put(Pair.newPair(IntDimEnc.ENCODING_NAME, value.getCurrentVersion()), value);
+ }
+ {
+ IntegerDimEnc.Factory value = new IntegerDimEnc.Factory();
+ map.put(Pair.newPair(IntegerDimEnc.ENCODING_NAME, value.getCurrentVersion()), value);
+ }
+ {
+ IntegerDimEncV2.Factory value = new IntegerDimEncV2.Factory();
+ map.put(Pair.newPair(IntegerDimEncV2.ENCODING_NAME, value.getCurrentVersion()), value);
+ }
+ {
+ FixedLenHexDimEnc.Factory value = new FixedLenHexDimEnc.Factory();
+ map.put(Pair.newPair(FixedLenHexDimEnc.ENCODING_NAME, value.getCurrentVersion()), value);
+ }
+ {
+ DateDimEnc.Factory value = new DateDimEnc.Factory();
+ map.put(Pair.newPair(DateDimEnc.ENCODING_NAME, value.getCurrentVersion()), value);
+ }
+ {
+ TimeDimEnc.Factory value = new TimeDimEnc.Factory();
+ map.put(Pair.newPair(TimeDimEnc.ENCODING_NAME, value.getCurrentVersion()), value);
+ }
+ {
+ BooleanDimEnc.Factory value = new BooleanDimEnc.Factory();
+ map.put(Pair.newPair(BooleanDimEnc.ENCODING_NAME, value.getCurrentVersion()), value);
+ }
// custom encodings
String[] clsNames = KylinConfig.getInstanceFromEnv().getCubeDimensionCustomEncodingFactories();
for (String clsName : clsNames) {
try {
DimensionEncodingFactory factory = (DimensionEncodingFactory) ClassUtil.newInstance(clsName);
- map.put(factory.getSupportedEncodingName(), factory);
+ map.put(Pair.newPair(factory.getSupportedEncodingName(), factory.getCurrentVersion()), factory);
} catch (Exception ex) {
logger.error("Failed to init dimension encoding factory " + clsName, ex);
}
@@ -96,9 +146,13 @@ public abstract class DimensionEncodingFactory {
}
}
- /** Return the supported encoding name, corresponds to RowKeyColDesc.encodingName */
+ /**
+ * Return the supported encoding name, corresponds to RowKeyColDesc.encodingName
+ */
abstract public String getSupportedEncodingName();
- /** Create a DimensionEncoding instance, with inputs corresponding to RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs */
+ /**
+ * Create a DimensionEncoding instance, with inputs corresponding to RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs
+ */
abstract public DimensionEncoding createDimensionEncoding(String encodingName, String[] args);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/caf7040f/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index f34c57c..14ce85c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -57,6 +57,7 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
public static final String DATATYPE_TOPN = "topn";
public static final String CONFIG_ENCODING_PREFIX = "topn.encoding.";
+ public static final String CONFIG_ENCODING_VERSION_PREFIX = "topn.encoding_version.";
public static final String CONFIG_AGG = "topn.aggregation";
public static final String CONFIG_ORDER = "topn.order";
@@ -413,11 +414,20 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
for (int i = 0; i < literalCols.size(); i++) {
TblColRef colRef = literalCols.get(i);
String encoding = function.getConfiguration().get(TopNMeasureType.CONFIG_ENCODING_PREFIX + colRef.getName());
+ String encodingVersionStr = function.getConfiguration().get(TopNMeasureType.CONFIG_ENCODING_VERSION_PREFIX + colRef.getName());
if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) {
dimensionEncodings[i] = new DictionaryDimEnc(dictionaryMap.get(colRef));
} else {
+ int encodingVersion = 1;
+ if (!StringUtils.isEmpty(encodingVersionStr)) {
+ try {
+ encodingVersion = Integer.parseInt(encodingVersionStr);
+ } catch (NumberFormatException e) {
+ throw new RuntimeException(TopNMeasureType.CONFIG_ENCODING_VERSION_PREFIX + colRef.getName() + " has to be an integer");
+ }
+ }
Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding);
- dimensionEncodings[i] = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1]);
+ dimensionEncodings[i] = DimensionEncodingFactory.create((String) encodingConf[0], (String[]) encodingConf[1], encodingVersion);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/caf7040f/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 5ca7cb5..e1aa17a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -24,7 +24,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.UUID;
import org.apache.commons.lang.StringUtils;
@@ -74,7 +73,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import com.google.common.collect.Maps;
/**
* CubeController is defined as Restful API entrance for UI.
@@ -112,12 +111,13 @@ public class CubeController extends BasicController {
@RequestMapping(value = "validEncodings", method = { RequestMethod.GET })
@ResponseBody
- public Set<String> getValidEncodings() {
- Set<String> encodings;
+ public Map<String, Integer> getValidEncodings() {
+ Map<String, Integer> encodings;
try {
encodings = DimensionEncodingFactory.getValidEncodings();
} catch (Exception e) {
- return Sets.newTreeSet();
+ logger.error("Error when getting valid encodings", e);
+ return Maps.newHashMap();
}
return encodings;
}
@@ -346,7 +346,7 @@ public class CubeController extends BasicController {
CubeDesc cubeDesc = cube.getDescriptor();
CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
-
+
KylinConfig config = cubeService.getConfig();
newCubeDesc.setName(newCubeName);
newCubeDesc.setEngineType(config.getDefaultCubeEngine());
@@ -614,7 +614,6 @@ public class CubeController extends BasicController {
}
-
/**
* Initiate the very beginning of a streaming cube. Will seek the latest offests of each partition from streaming
* source (kafka) and record in the cube descriptor; In the first build job, it will use these offests as the start point.