You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/11/30 07:03:07 UTC
[1/2] incubator-kylin git commit: KYLIN-980 FactDistinctColumnsJob to
support high cardinality columns
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging 623635d4a -> 43bed4b15
KYLIN-980 FactDistinctColumnsJob to support high cardinality columns
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/43bed4b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/43bed4b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/43bed4b1
Branch: refs/heads/2.x-staging
Commit: 43bed4b1529e939ece9903e559cf217a33586901
Parents: 05e1f61
Author: shaofengshi <sh...@apache.org>
Authored: Mon Nov 30 14:01:35 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Nov 30 14:02:38 2015 +0800
----------------------------------------------------------------------
build/bin/find-hive-dependency.sh | 8 +-
.../kylin/engine/mr/common/CuboidStatsUtil.java | 25 ++++---
.../mr/steps/FactDistinctColumnsCombiner.java | 35 ++-------
.../engine/mr/steps/FactDistinctColumnsJob.java | 3 +-
.../mr/steps/FactDistinctColumnsMapperBase.java | 5 +-
.../mr/steps/FactDistinctColumnsReducer.java | 78 ++++++++++----------
.../mr/steps/FactDistinctHiveColumnsMapper.java | 22 ++++--
7 files changed, 84 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/43bed4b1/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index e2c8ffa..47f0fbc 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -40,10 +40,10 @@ done
# in some versions of hive hcatalog is not in hive's classpath, find it separately
if [ -z "$HCAT_HOME" ]
then
- echo "HCAT_HOME not found, try to find hcatalog path from hive home"
- hive_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
- if [ -d "${hive_home}/hive-hcatalog" ]; then
- hcatalog_home=${hive_home}/hive-hcatalog
+ echo "HCAT_HOME not found, try to find hcatalog path from hadoop home"
+ hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
+ if [ -d "${hadoop_home}/hive-hcatalog" ]; then
+ hcatalog_home=${hadoop_home}/hive-hcatalog
elif [ -d "${hadoop_home}/hive/hcatalog" ]; then
hcatalog_home=${hadoop_home}/hive/hcatalog
else
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/43bed4b1/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
index 10c82c3..3f46128 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsUtil.java
@@ -18,13 +18,7 @@
package org.apache.kylin.engine.mr.common;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
@@ -34,20 +28,27 @@ import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.kv.RowConstants;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
public class CuboidStatsUtil {
- public static void writeCuboidStatistics(Configuration conf, Path outputPath, Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage) throws IOException {
+ public static void writeCuboidStatistics(Configuration conf, Path outputPath, Map<Long, HyperLogLogPlusCounter> cuboidHLLMap, int samplingPercentage) throws IOException {
Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION);
- SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(seqFilePath), SequenceFile.Writer.keyClass(LongWritable.class), SequenceFile.Writer.valueClass(BytesWritable.class));
List<Long> allCuboids = new ArrayList<Long>();
allCuboids.addAll(cuboidHLLMap.keySet());
Collections.sort(allCuboids);
- // persist the sample percentage with key 0
- writer.append(new LongWritable(0l), new BytesWritable(Bytes.toBytes(samplingPercentage)));
ByteBuffer valueBuf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
+ SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(seqFilePath), SequenceFile.Writer.keyClass(LongWritable.class), SequenceFile.Writer.valueClass(BytesWritable.class));
try {
+ // persist the sample percentage with key 0
+ writer.append(new LongWritable(0l), new BytesWritable(Bytes.toBytes(samplingPercentage)));
for (long i : allCuboids) {
valueBuf.clear();
cuboidHLLMap.get(i).writeRegisters(valueBuf);
@@ -55,7 +56,7 @@ public class CuboidStatsUtil {
writer.append(new LongWritable(i), new BytesWritable(valueBuf.array(), valueBuf.limit()));
}
} finally {
- writer.close();
+ IOUtils.closeQuietly(writer);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/43bed4b1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
index b59ac07..98ab199 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsCombiner.java
@@ -18,21 +18,15 @@
package org.apache.kylin.engine.mr.steps;
-import java.io.IOException;
-import java.util.HashSet;
-
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.engine.mr.KylinReducer;
+import java.io.IOException;
+
/**
* @author yangli9
*/
-public class FactDistinctColumnsCombiner extends KylinReducer<LongWritable, Text, LongWritable, Text> {
-
- private Text outputValue = new Text();
+public class FactDistinctColumnsCombiner extends KylinReducer<Text, Text, Text, Text> {
@Override
protected void setup(Context context) throws IOException {
@@ -40,24 +34,11 @@ public class FactDistinctColumnsCombiner extends KylinReducer<LongWritable, Text
}
@Override
- public void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
-
- if (key.get() >= 0) {
- HashSet<ByteArray> set = new HashSet<ByteArray>();
- for (Text textValue : values) {
- ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
- set.add(value);
- }
-
- for (ByteArray value : set) {
- outputValue.set(value.array(), value.offset(), value.length());
- context.write(key, outputValue);
- }
- } else {
- // for hll, each key only has one output, no need to do local combine;
- outputValue.set(values.iterator().next().getBytes());
- context.write(key, outputValue);
- }
+ public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+
+ // for hll, each key only has one output, no need to do local combine;
+ // for normal col, values are empty text
+ context.write(key, values.iterator().next());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/43bed4b1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
index 6bcef28..83bced1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.commons.cli.Options;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
@@ -108,7 +107,7 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
job.setMapperClass(FactDistinctHiveColumnsMapper.class);
job.setCombinerClass(FactDistinctColumnsCombiner.class);
- job.setMapOutputKeyClass(LongWritable.class);
+ job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/43bed4b1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index b97c88a..2279b8b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -1,7 +1,6 @@
package org.apache.kylin.engine.mr.steps;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
@@ -26,7 +25,7 @@ import java.util.List;
/**
*/
-public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VALUEIN, LongWritable, Text> {
+public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<KEYIN, VALUEIN, Text, Text> {
protected String cubeName;
protected CubeInstance cube;
@@ -37,7 +36,7 @@ public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends KylinMapper<K
protected ArrayList<Integer> factDictCols;
protected IMRTableInputFormat flatTableInputFormat;
- protected LongWritable outputKey = new LongWritable();
+ protected Text outputKey = new Text();
protected Text outputValue = new Text();
protected int errorRecordCounter = 0;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/43bed4b1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index ddd366c..71adaa9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -21,18 +21,17 @@ package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
import java.util.*;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.MemoryBudgetController;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
@@ -48,9 +47,9 @@ import com.google.common.collect.Maps;
/**
* @author yangli9
*/
-public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text, NullWritable, Text> {
+public class FactDistinctColumnsReducer extends KylinReducer<Text, Text, NullWritable, Text> {
- private List<TblColRef> columnList = new ArrayList<TblColRef>();
+ private List<TblColRef> columnList = new ArrayList();
private boolean collectStatistics = false;
private String statisticsOutput = null;
private List<Long> baseCuboidRowCountInMappers;
@@ -58,7 +57,9 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
protected long baseCuboidId;
protected CubeDesc cubeDesc;
private long totalRowsBeforeMerge = 0;
- private int SAMPING_PERCENTAGE = 5;
+ private int SAMPING_PERCENTAGE = 100;
+ private List<ByteArray> colValues;
+ private long currentColIndex = -1;
@Override
protected void setup(Context context) throws IOException {
@@ -77,35 +78,29 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
if (collectStatistics) {
baseCuboidRowCountInMappers = Lists.newArrayList();
cuboidHLLMap = Maps.newHashMap();
- SAMPING_PERCENTAGE = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, "5"));
+ SAMPING_PERCENTAGE = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, "100"));
}
+
+ colValues = Lists.newArrayList();
}
@Override
- public void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
-
- if (key.get() >= 0) {
- TblColRef col = columnList.get((int) key.get());
-
- HashSet<ByteArray> set = new HashSet<ByteArray>();
- int count = 0;
- for (Text textValue : values) {
- ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength()));
- set.add(value);
- count++;
- if (count % 10000 == 0 && MemoryBudgetController.getSystemAvailMB() < 100) {
- outputDistinctValues(col, set, context);
- set.clear();
+ public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+
+ long colIndex = Bytes.toLong(key.getBytes(), 0, Bytes.SIZEOF_LONG);
+ if (colIndex >= 0) {
+ if (colIndex != currentColIndex || colValues.size() == 1000000) { //spill every 1 million
+ if (colValues.size() > 0) {
+ System.out.println("spill values to disk...");
+ outputDistinctValues(columnList.get((int) currentColIndex), colValues, context);
+ colValues.clear();
}
+ currentColIndex = colIndex;
}
-
- if (set.isEmpty() == false) {
- outputDistinctValues(col, set, context);
- }
-
+ colValues.add(new ByteArray(Bytes.copy(key.getBytes(), Bytes.SIZEOF_LONG, key.getLength() - Bytes.SIZEOF_LONG)));
} else {
// for hll
- long cuboidId = 0 - key.get();
+ long cuboidId = 0 - colIndex;
for (Text value : values) {
HyperLogLogPlusCounter hll = new HyperLogLogPlusCounter(14);
@@ -128,31 +123,40 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
}
- private void outputDistinctValues(TblColRef col, Set<ByteArray> set, Context context) throws IOException {
+ private void outputDistinctValues(TblColRef col, Collection<ByteArray> values, Context context) throws IOException {
final Configuration conf = context.getConfiguration();
final FileSystem fs = FileSystem.get(conf);
final String outputPath = conf.get(BatchConstants.OUTPUT_PATH);
final Path outputFile = new Path(outputPath, col.getName());
- FSDataOutputStream out;
- if (fs.exists(outputFile)) {
- out = fs.append(outputFile);
- } else {
- out = fs.create(outputFile);
- }
+ FSDataOutputStream out = null;
try {
- for (ByteArray value : set) {
+ if (fs.exists(outputFile)) {
+ out = fs.append(outputFile);
+ System.out.println("append file " + outputFile);
+ } else {
+ out = fs.create(outputFile);
+ System.out.println("create file " + outputFile);
+ }
+
+ for (ByteArray value: values) {
out.write(value.array(), value.offset(), value.length());
out.write('\n');
}
} finally {
- out.close();
+ IOUtils.closeQuietly(out);
+ IOUtils.closeQuietly(fs);
}
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
+ if (colValues.size() > 0) {
+ outputDistinctValues(columnList.get((int)currentColIndex), colValues, context);
+ colValues.clear();
+ }
+
//output the hll info;
if (collectStatistics) {
writeMapperAndCuboidStatistics(context); // for human check
@@ -168,7 +172,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
try {
String msg;
- List<Long> allCuboids = new ArrayList<Long>();
+ List<Long> allCuboids = Lists.newArrayList();
allCuboids.addAll(cuboidHLLMap.keySet());
Collections.sort(allCuboids);
@@ -204,7 +208,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<LongWritable, Text,
}
} finally {
- out.close();
+ IOUtils.closeQuietly(out);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/43bed4b1/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
index e43d5d1..eb67ad9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import com.google.common.collect.Maps;
+import org.apache.hadoop.io.Text;
import org.apache.kylin.common.hll.HyperLogLogPlusCounter;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
@@ -55,12 +56,13 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
private int rowCount = 0;
private int SAMPING_PERCENTAGE = 5;
private ByteArray[] row_hashcodes = null;
+ private ByteBuffer keyBuffer;
+ private static final Text EMPTY_TEXT = new Text();
@Override
protected void setup(Context context) throws IOException {
super.setup(context);
-
-
+ keyBuffer = ByteBuffer.allocate(4096);
collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
if (collectStatistics) {
SAMPING_PERCENTAGE = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT, "5"));
@@ -114,13 +116,19 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
String[] row = flatTableInputFormat.parseMapperInput(record);
try {
for (int i = 0; i < factDictCols.size(); i++) {
- outputKey.set((long) factDictCols.get(i));
+// outputKey.set(Bytes.toBytes((long)factDictCols.get(i)));
String fieldValue = row[dictionaryColumnIndex[i]];
if (fieldValue == null)
continue;
- byte[] bytes = Bytes.toBytes(fieldValue);
- outputValue.set(bytes, 0, bytes.length);
- context.write(outputKey, outputValue);
+// outputValue.set(bytes, 0, bytes.length);
+
+ keyBuffer.clear();
+ keyBuffer.putLong((long)factDictCols.get(i));
+ keyBuffer.put(Bytes.toBytes(fieldValue));
+ outputKey.set(keyBuffer.array(), 0, keyBuffer.position());
+ context.write(outputKey, EMPTY_TEXT);
+
+// context.write(outputKey, outputValue);
}
} catch (Exception ex) {
handleErrorRecord(row, ex);
@@ -166,7 +174,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
HyperLogLogPlusCounter hll;
for (int i = 0; i < cuboidIds.length; i++) {
hll = allCuboidsHLL[i];
- outputKey.set(0 - cuboidIds[i]);
+ outputKey.set(Bytes.toBytes(0 - cuboidIds[i]));
hllBuf.clear();
hll.writeRegisters(hllBuf);
outputValue.set(hllBuf.array(), 0, hllBuf.position());
[2/2] incubator-kylin git commit: Update test metadata,
make them be runable from UI
Posted by sh...@apache.org.
Update test metadata, make them be runable from UI
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/05e1f612
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/05e1f612
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/05e1f612
Branch: refs/heads/2.x-staging
Commit: 05e1f612afca2f6097d54c8f629f5971cafcb32b
Parents: 623635d
Author: shaofengshi <sh...@apache.org>
Authored: Fri Nov 27 14:56:18 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Nov 30 14:02:38 2015 +0800
----------------------------------------------------------------------
.../model/validation/rule/FunctionRule.java | 4 ++
.../cube/test_kylin_cube_with_slr_empty.json | 1 +
.../cube/test_kylin_cube_without_slr_empty.json | 1 +
.../localmeta/project/default.json | 43 ++++++++++++++++++--
.../kylin/rest/controller/CubeController.java | 8 +---
.../apache/kylin/rest/service/CubeService.java | 25 +++---------
.../kylin/rest/service/CubeServiceTest.java | 3 --
7 files changed, 52 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e1f612/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
index 1920fc7..5f400d7 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/FunctionRule.java
@@ -122,6 +122,10 @@ public class FunctionRule implements IValidatorRule<CubeDesc> {
if (rtype.isNumberFamily() == false) {
context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.NUMBER_FAMILY);
}
+ } else if (funcDesc.isTopN()) {
+ if (rtype.isTopN() == false) {
+ context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of topn");
+ }
} else {
if (StringUtils.equalsIgnoreCase(KylinConfig.getInstanceFromEnv().getProperty(KEY_IGNORE_UNKNOWN_FUNC, "false"), "false")) {
context.addResult(ResultLevel.ERROR, "Unrecognized function: [" + func + "]");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e1f612/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json
index 70d89ea..69032bc 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_empty.json
@@ -6,5 +6,6 @@
"version" : null,
"descriptor" : "test_kylin_cube_with_slr_desc",
"segments" : [ ],
+ "status" : "DISABLED",
"create_time" : null
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e1f612/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json
index 6bd566a..fe3c4d7 100644
--- a/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_without_slr_empty.json
@@ -6,5 +6,6 @@
"version" : null,
"descriptor" : "test_kylin_cube_without_slr_desc",
"segments" : [ ],
+ "status" : "DISABLED",
"create_time" : null
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e1f612/examples/test_case_data/localmeta/project/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json
index e4118ca..aeb79bd 100644
--- a/examples/test_case_data/localmeta/project/default.json
+++ b/examples/test_case_data/localmeta/project/default.json
@@ -3,26 +3,61 @@
"name": "default",
"realizations": [
{
- "name": "test_kylin_with_slr_model_desc",
+ "name": "test_kylin_cube_with_slr_empty",
"type": "CUBE",
"realization": "test_kylin_cube_with_slr_empty"
},
{
- "name": "test_kylin_without_slr_model_desc",
+ "name": "test_kylin_cube_without_slr_empty",
"type": "CUBE",
"realization": "test_kylin_cube_without_slr_empty"
},
{
- "name": "test_kylin_with_slr_left_join_model_desc",
+ "name": "test_kylin_cube_with_slr_left_join_empty",
"type": "CUBE",
"realization": "test_kylin_cube_with_slr_left_join_empty"
},
{
- "name": "test_kylin_without_slr_left_join_model_desc",
+ "name": "test_kylin_cube_without_slr_left_join_empty",
"type": "CUBE",
"realization": "test_kylin_cube_without_slr_left_join_empty"
},
{
+ "name": "test_kylin_cube_with_slr_1_new_segment",
+ "type": "CUBE",
+ "realization": "test_kylin_cube_with_slr_1_new_segment"
+ },
+ {
+ "name": "test_kylin_cube_with_slr_left_join_ready",
+ "type": "CUBE",
+ "realization": "test_kylin_cube_with_slr_left_join_ready"
+ },
+ {
+ "name": "test_kylin_cube_with_slr_ready",
+ "type": "CUBE",
+ "realization": "test_kylin_cube_with_slr_ready"
+ },
+ {
+ "name": "test_kylin_cube_with_slr_ready_2_segments",
+ "type": "CUBE",
+ "realization": "test_kylin_cube_with_slr_ready_2_segments"
+ },
+ {
+ "name": "test_kylin_cube_without_slr_left_join_ready",
+ "type": "CUBE",
+ "realization": "test_kylin_cube_without_slr_left_join_ready"
+ },
+ {
+ "name": "test_kylin_cube_without_slr_left_join_ready_2_segments",
+ "type": "CUBE",
+ "realization": "test_kylin_cube_without_slr_left_join_ready_2_segments"
+ },
+ {
+ "name": "test_kylin_cube_without_slr_ready",
+ "type": "CUBE",
+ "realization": "test_kylin_cube_without_slr_ready"
+ },
+ {
"name": "test_kylin_ii_left_join",
"type": "INVERTED_INDEX",
"realization": "test_kylin_ii_left_join"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e1f612/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index b0c3387..1427ff2 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -328,7 +328,7 @@ public class CubeController extends BasicController {
cubeRequest.setMessage("CubeDesc is null.");
return cubeRequest;
}
- String name = CubeService.getCubeNameFromDesc(desc.getName());
+ String name = cubeRequest.getCubeName();
if (StringUtils.isEmpty(name)) {
logger.info("Cube name should not be empty.");
throw new BadRequestException("Cube name should not be empty.");
@@ -431,12 +431,6 @@ public class CubeController extends BasicController {
return cubeRequest;
}
- //cube renaming:
- if (!cubeRequest.getCubeName().equalsIgnoreCase(CubeService.getCubeNameFromDesc(desc.getName()))) {
- deleteCube(cubeRequest.getCubeName());
- saveCubeDesc(cubeRequest);
- }
-
try {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName());
String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e1f612/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 6670de1..5e1a3d9 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -289,29 +289,16 @@ public class CubeService extends BasicService {
}
public boolean isCubeDescEditable(CubeDesc cd) {
- String cubeName = getCubeNameFromDesc(cd.getName());
- CubeInstance cube = getCubeManager().getCube(cubeName);
- if (cube == null) {
- return true;
- }
- if (cube.getSegments().size() != 0) {
- return false;
+ List<CubeInstance> cubes = getCubeManager().getCubesByDesc(cd.getName());
+ for (CubeInstance cube : cubes) {
+ if (cube.getSegments().size() != 0) {
+ logger.debug("cube '" + cube.getName() + " has " + cube.getSegments().size() + " segments, couldn't edit cube desc.");
+ return false;
+ }
}
return true;
}
- public static String getCubeDescNameFromCube(String cubeName) {
- return cubeName + DESC_SUFFIX;
- }
-
- public static String getCubeNameFromDesc(String descName) {
- if (descName.toLowerCase().endsWith(DESC_SUFFIX)) {
- return descName.substring(0, descName.toLowerCase().indexOf(DESC_SUFFIX));
- } else {
- return descName;
- }
- }
-
/**
* Stop all jobs belonging to this cube and clean out all segments
*
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/05e1f612/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
index 226380c..d86935e 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
@@ -48,9 +48,6 @@ public class CubeServiceTest extends ServiceTestBase {
Assert.assertNotNull(cubeService.getMetadataManager());
Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
- Assert.assertTrue(CubeService.getCubeDescNameFromCube("testCube").equals("testCube_desc"));
- Assert.assertTrue(CubeService.getCubeNameFromDesc("testCube_desc").equals("testCube"));
-
List<CubeInstance> cubes = cubeService.getCubes(null, null, null, null, null);
Assert.assertNotNull(cubes);
CubeInstance cube = cubes.get(0);