You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/09/10 08:51:39 UTC
[kylin] branch master updated (7a77141 -> bffaa8a)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 7a77141 KYLIN-3258 No check for duplicate cube name when creating a hybrid cube
new 37a4966 KYLIN-3529 Show more error message
new d164225 KYLIN-3534 Don't compress fact distinct output file
new 5faddf6 KYLIN-3527 Hybrid allows 0,1 cube
new bffaa8a KYLIN-3539 Hybrid segment overlap check
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../kylin/common/util/AbstractApplication.java | 2 +-
.../kylin/engine/spark/SparkFactDistinct.java | 7 +++-
.../org/apache/kylin/rest/job/HybridCubeCLI.java | 34 ++++++++-------
.../org/apache/kylin/tool/HybridCubeCLITest.java | 48 ++++++++++++++++++++++
4 files changed, 74 insertions(+), 17 deletions(-)
[kylin] 01/04: KYLIN-3529 Show more error message
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 37a4966db41c775dfc48cdd4a4a244f03a18a96c
Author: chao long <wa...@qq.com>
AuthorDate: Wed Sep 5 09:55:10 2018 +0800
KYLIN-3529 Show more error message
---
.../src/main/java/org/apache/kylin/common/util/AbstractApplication.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java b/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java
index ab1e691..7275e10 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java
@@ -39,7 +39,7 @@ public abstract class AbstractApplication {
optionsHelper.printUsage(this.getClass().getName(), getOptions());
throw new RuntimeException("error parsing args", e);
} catch (Exception e) {
- throw new RuntimeException("error execute " + this.getClass().getName(), e);
+ throw new RuntimeException("error execute " + this.getClass().getName() + ". Root cause: " + e.getMessage(), e);
}
}
}
[kylin] 02/04: KYLIN-3534 Don't compress fact distinct output file
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d164225e91fa59aa1dd06d2ddf64e6acc1035a3f
Author: chao long <wa...@qq.com>
AuthorDate: Wed Sep 5 09:55:44 2018 +0800
KYLIN-3534 Don't compress fact distinct output file
---
.../main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
index d0ba5d5..632a169 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
@@ -209,19 +209,22 @@ public class SparkFactDistinct extends AbstractApplication implements Serializab
NullWritable.class, LongWritable.class);
FileOutputFormat.setOutputPath(job, new Path(outputPath));
+ FileOutputFormat.setCompressOutput(job, false);
// prevent to create zero-sized default output
LazyOutputFormat.setOutputFormatClass(job, SequenceFileOutputFormat.class);
+
MultipleOutputsRDD multipleOutputsRDD = MultipleOutputsRDD.rddToMultipleOutputsRDD(outputRDD);
multipleOutputsRDD.saveAsNewAPIHadoopDatasetWithMultipleOutputs(job.getConfiguration());
- logger.info("Map input records={}", recordRDD.count());
+ long recordCount = recordRDD.count();
+ logger.info("Map input records={}", recordCount);
logger.info("HDFS Read: {} HDFS Write", bytesWritten.value());
Map<String, String> counterMap = Maps.newHashMap();
- counterMap.put(ExecutableConstants.SOURCE_RECORDS_COUNT, String.valueOf(recordRDD.count()));
+ counterMap.put(ExecutableConstants.SOURCE_RECORDS_COUNT, String.valueOf(recordCount));
counterMap.put(ExecutableConstants.SOURCE_RECORDS_SIZE, String.valueOf(bytesWritten.value()));
// save counter to hdfs
[kylin] 04/04: KYLIN-3539 Hybrid segment overlap check
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit bffaa8a26a2d93b3aa62326484bc31a8717a7c0b
Author: chao long <wa...@qq.com>
AuthorDate: Mon Sep 10 16:00:46 2018 +0800
KYLIN-3539 Hybrid segment overlap check
---
.../org/apache/kylin/rest/job/HybridCubeCLI.java | 30 +++++++++-----
.../org/apache/kylin/tool/HybridCubeCLITest.java | 48 ++++++++++++++++++++++
2 files changed, 68 insertions(+), 10 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
index 303e000..443eba6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
+import java.util.Collections;
import java.util.List;
import org.apache.commons.cli.Option;
@@ -38,6 +39,8 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.DataModelManager;
+import org.apache.kylin.metadata.model.SegmentRange;
+import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -46,6 +49,8 @@ import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
/**
* 1. Create new HybridCube
* bin/kylin.sh org.apache.kylin.tool.HybridCubeCLI -action create -name hybrid_name -project project_name -model model_name -cubes cube1,cube2
@@ -202,23 +207,28 @@ public class HybridCubeCLI extends AbstractApplication {
}
private void checkSegmentOffset(List<RealizationEntry> realizationEntries) {
- long lastOffset = -1;
+ List<SegmentRange> segmentRanges = Lists.newArrayList();
+
for (RealizationEntry entry : realizationEntries) {
if (entry.getType() != RealizationType.CUBE) {
throw new IllegalArgumentException("Wrong realization type: " + entry.getType() + ", only cube supported. ");
}
CubeInstance cubeInstance = cubeManager.getCube(entry.getRealization());
- CubeSegment segment = cubeInstance.getLastSegment();
- if (segment == null)
- continue;
- if (lastOffset == -1) {
- lastOffset = (Long) segment.getSegRange().end.v;
- } else {
- if (lastOffset > (Long) segment.getSegRange().start.v) {
- throw new IllegalArgumentException("Segments has overlap, could not hybrid. Last Segment End: " + lastOffset + ", Next Segment Start: " + segment.getSegRange().start.v);
+ Segments<CubeSegment> segments = cubeInstance.getSegments();
+
+ for (CubeSegment segment : segments) {
+ segmentRanges.add(segment.getSegRange());
+ }
+ }
+
+ if (segmentRanges.size() >= 2) {
+ Collections.sort(segmentRanges);
+
+ for (int i = 0; i < segmentRanges.size() - 1; i++) {
+ if (segmentRanges.get(i).overlaps(segmentRanges.get(i + 1))) {
+ throw new IllegalArgumentException("Segments has overlap, could not hybrid. First Segment Range: [" + segmentRanges.get(i).start.v + "," + segmentRanges.get(i).end.v + "], Second Segment Range: [" + segmentRanges.get(i + 1).start.v + "," + segmentRanges.get(i + 1).end.v + "]");
}
- lastOffset = (Long) segment.getSegRange().end.v;
}
}
}
diff --git a/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
index f4ed595..99b8319 100644
--- a/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
+++ b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
@@ -22,6 +22,9 @@ import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.storage.hybrid.HybridInstance;
@@ -30,12 +33,17 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.FixMethodOrder;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.junit.runners.MethodSorters;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class HybridCubeCLITest extends LocalFileMetadataTestCase {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
@Before
public void setUp() throws Exception {
this.createTestMetadata();
@@ -89,4 +97,44 @@ public class HybridCubeCLITest extends LocalFileMetadataTestCase {
Assert.assertEquals(0, ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).findProjects(RealizationType.HYBRID, "ssb_hybrid").size());
}
+ @Test
+ public void testSegmentOverlap() throws IOException {
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("Segments has overlap");
+
+ HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
+
+ CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
+ CubeInstance cube2 = cubeManager.getCube("ssb_cube2");
+
+ // 2012-01-01,2012-01-03
+ cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
+ // 2012-01-02,2012-01-04
+ cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325462400000L, 1325635200000L));
+
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
+ }
+
+ @Test
+ public void testSegmentNotOverlap() throws IOException {
+ HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
+
+ CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
+ CubeInstance cube2 = cubeManager.getCube("ssb_cube2");
+
+ // 2012-01-01,2012-01-03
+ cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
+ // 2012-01-03,2012-01-04
+ cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325548800000L, 1325635200000L));
+
+ // not throw exception
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
+ }
+
}
[kylin] 03/04: KYLIN-3527 Hybrid allows 0,1 cube
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 5faddf6eb5ba7e8bb682e1ee6fded4a7e89a2d92
Author: chao long <wa...@qq.com>
AuthorDate: Wed Sep 5 14:16:31 2018 +0800
KYLIN-3527 Hybrid allows 0,1 cube
---
.../src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java | 4 ----
1 file changed, 4 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
index e332e1d..303e000 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
@@ -202,10 +202,6 @@ public class HybridCubeCLI extends AbstractApplication {
}
private void checkSegmentOffset(List<RealizationEntry> realizationEntries) {
- if (realizationEntries == null || realizationEntries.size() == 0)
- throw new IllegalArgumentException("No realization found");
- if (realizationEntries.size() == 1)
- throw new IllegalArgumentException("Hybrid needs at least 2 cubes");
long lastOffset = -1;
for (RealizationEntry entry : realizationEntries) {
if (entry.getType() != RealizationType.CUBE) {