You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/06/25 03:56:50 UTC
[kylin] branch 2.6.x updated: KYLIN-3893 Add validation for the
encoding length and the type of encoding (#622)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.6.x by this push:
new a0463c4 KYLIN-3893 Add validation for the encoding length and the type of encoding (#622)
a0463c4 is described below
commit a0463c478b25f7a1f36d4bc255327bd0acf2ab0c
Author: Na Zhai <38...@users.noreply.github.com>
AuthorDate: Thu May 30 17:33:53 2019 +0800
KYLIN-3893 Add validation for the encoding length and the type of encoding (#622)
* KYLIN-3893 Add validation for the encoding length and the type of encoding
* KYLIN-3893 Code review
---
.../java/org/apache/kylin/cube/kv/CubeDimEncMap.java | 5 +++--
.../java/org/apache/kylin/dimension/BooleanDimEnc.java | 5 +++++
.../java/org/apache/kylin/dimension/DateDimEnc.java | 16 ++++++++++++++++
.../kylin/dimension/DimensionEncodingFactory.java | 17 +++++++++++++++++
.../java/org/apache/kylin/dimension/FixedLenDimEnc.java | 10 ++++++++++
.../org/apache/kylin/dimension/FixedLenHexDimEnc.java | 10 ++++++++++
.../main/java/org/apache/kylin/dimension/IntDimEnc.java | 10 ++++++++++
.../java/org/apache/kylin/dimension/IntegerDimEnc.java | 10 ++++++++++
.../apache/kylin/dimension/OneMoreByteVLongDimEnc.java | 10 ++++++++++
.../java/org/apache/kylin/dimension/TimeDimEnc.java | 5 +++++
.../apache/kylin/rest/controller/CubeController.java | 11 +++++++++++
11 files changed, 107 insertions(+), 2 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
index bd9554a..1de63a3 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
@@ -18,7 +18,8 @@
package org.apache.kylin.cube.kv;
-import com.google.common.collect.Maps;
+import java.util.Map;
+
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
@@ -32,7 +33,7 @@ import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
+import com.google.common.collect.Maps;
public class CubeDimEncMap implements IDimensionEncodingMap, java.io.Serializable {
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
index fbdb0bb..fb6ab95 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
@@ -60,6 +60,11 @@ public class BooleanDimEnc extends DimensionEncoding implements Serializable{
public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
return new BooleanDimEnc();
}
+
+ @Override
+ public boolean isValidDimensionEncoding(String encodingName, String[] args) {
+ return true;
+ }
};
// ============================================================================
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
index 6f06841..f6f4c2f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
@@ -44,6 +44,22 @@ public class DateDimEnc extends AbstractDateDimEnc implements Serializable {
public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
return new DateDimEnc(args);
}
+
+ @Override
+ public boolean isValidDimensionEncoding(String encodingName, String[] args) {
+ try {
+ if (args != null && args.length == 1) {
+ DataType datatype = DataType.getType(args[0]);
+ if (datatype == null) {
+ return false;
+ }
+ }
+ } catch (Exception e) {
+ return false;
+ }
+
+ return true;
+ }
};
private static class DateDimValueCodec implements IValueCodec {
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
index 2750463..8f0e2d0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncodingFactory.java
@@ -97,6 +97,18 @@ public abstract class DimensionEncodingFactory {
});
}
+ public static boolean isValidEncoding(final String encodingName, String[] args, int version) {
+ if (factoryMap == null)
+ initFactoryMap();
+
+ DimensionEncodingFactory factory = factoryMap.get(Pair.newPair(encodingName, version));
+ if (factory == null) {
+ return true;
+ }
+
+ return factory.isValidDimensionEncoding(encodingName, args);
+ }
+
private static synchronized void initFactoryMap() {
if (factoryMap == null) {
Map<Pair<String, Integer>, DimensionEncodingFactory> map = Maps.newConcurrentMap();
@@ -155,4 +167,9 @@ public abstract class DimensionEncodingFactory {
* Create a DimensionEncoding instance, with inputs corresponding to RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs
*/
abstract public DimensionEncoding createDimensionEncoding(String encodingName, String[] args);
+
+ /**
+ * Validate the inputs of RowKeyColDesc.encodingName and RowKeyColDesc.encodingArgs for a DimensionEncoding instance
+ */
+ abstract public boolean isValidDimensionEncoding(String encodingName, String[] args);
}
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
index 9ce1577..be6b5bd 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
@@ -50,6 +50,16 @@ public class FixedLenDimEnc extends DimensionEncoding implements Serializable{
public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
return new FixedLenDimEnc(Integer.parseInt(args[0]));
}
+
+ @Override
+ public boolean isValidDimensionEncoding(String encodingName, String[] args) {
+ try {
+ int length = Integer.parseInt(args[0]);
+ return length >= 1;
+ } catch (Exception e) {
+ return false;
+ }
+ }
};
// ============================================================================
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
index 1d7e3c9..f2fee8f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
@@ -112,6 +112,16 @@ public class FixedLenHexDimEnc extends DimensionEncoding implements Serializable
public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
return new FixedLenHexDimEnc(Integer.parseInt(args[0]));
}
+
+ @Override
+ public boolean isValidDimensionEncoding(String encodingName, String[] args) {
+ try {
+ int length = Integer.parseInt(args[0]);
+ return length >= 1;
+ } catch (Exception e) {
+ return false;
+ }
+ }
}
// ============================================================================
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
index 6ce003b..d3a5103 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
@@ -54,6 +54,16 @@ public class IntDimEnc extends DimensionEncoding implements Serializable {
public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
return new IntDimEnc(Integer.parseInt(args[0]));
}
+
+ @Override
+ public boolean isValidDimensionEncoding(String encodingName, String[] args) {
+ try {
+ int length = Integer.parseInt(args[0]);
+ return length >= 1 && length < CAP.length;
+ } catch (Exception e) {
+ return false;
+ }
+ }
};
// ============================================================================
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index e024696..7abd40c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -64,6 +64,16 @@ public class IntegerDimEnc extends DimensionEncoding implements Serializable{
public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
return new IntegerDimEnc(Integer.parseInt(args[0]));
}
+
+ @Override
+ public boolean isValidDimensionEncoding(String encodingName, String[] args) {
+ try {
+ int length = Integer.parseInt(args[0]);
+ return length >= 1 && length < CAP.length;
+ } catch (Exception e) {
+ return false;
+ }
+ }
};
// ============================================================================
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
index d998f44..c3e36a0 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
@@ -59,6 +59,16 @@ public class OneMoreByteVLongDimEnc extends DimensionEncoding implements Seriali
public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
return new OneMoreByteVLongDimEnc(Integer.parseInt(args[0]));
}
+
+ @Override
+ public boolean isValidDimensionEncoding(String encodingName, String[] args) {
+ try {
+ int length = Integer.parseInt(args[0]);
+ return length >= 1 && length < CAP.length;
+ } catch (Exception e) {
+ return false;
+ }
+ }
};
// ============================================================================
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
index dcc99dd..c9a96fc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
@@ -40,6 +40,11 @@ public class TimeDimEnc extends AbstractDateDimEnc implements Serializable {
public DimensionEncoding createDimensionEncoding(String encodingName, String[] args) {
return new TimeDimEnc();
}
+
+ @Override
+ public boolean isValidDimensionEncoding(String encodingName, String[] args) {
+ return true;
+ }
};
public TimeDimEnc() {
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index c3f45a6..c7626b8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -45,6 +45,7 @@ import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
import org.apache.kylin.cube.model.RowKeyColDesc;
+import org.apache.kylin.dimension.DimensionEncoding;
import org.apache.kylin.dimension.DimensionEncodingFactory;
import org.apache.kylin.engine.mr.common.CuboidStatsReaderUtil;
import org.apache.kylin.job.JobInstance;
@@ -625,6 +626,16 @@ public class CubeController extends BasicController {
throw new BadRequestException(
"the number of input measure and the number of measure defined in cubedesc are not consistent");
}
+
+ for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
+ Object[] encodingConf = DimensionEncoding.parseEncodingConf(rowKeyColDesc.getEncoding());
+ String encodingName = (String) encodingConf[0];
+ String[] encodingArgs = (String[]) encodingConf[1];
+
+ if (!DimensionEncodingFactory.isValidEncoding(encodingName, encodingArgs, rowKeyColDesc.getEncodingVersion())) {
+ throw new BadRequestException("Illegal row key column desc: " + rowKeyColDesc);
+ }
+ }
}
/**