You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@kylin.apache.org by GitBox <gi...@apache.org> on 2018/09/10 08:51:38 UTC
[GitHub] shaofengshi closed pull request #222: Bug fix 2.5.0
shaofengshi closed pull request #222: Bug fix 2.5.0
URL: https://github.com/apache/kylin/pull/222
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java b/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java
index ab1e691337..7275e10da3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/AbstractApplication.java
@@ -39,7 +39,7 @@ public final void execute(String[] args) {
optionsHelper.printUsage(this.getClass().getName(), getOptions());
throw new RuntimeException("error parsing args", e);
} catch (Exception e) {
- throw new RuntimeException("error execute " + this.getClass().getName(), e);
+ throw new RuntimeException("error execute " + this.getClass().getName() + ". Root cause: " + e.getMessage(), e);
}
}
}
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
index d0ba5d5cc3..632a169400 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkFactDistinct.java
@@ -209,19 +209,22 @@ protected void execute(OptionsHelper optionsHelper) throws Exception {
NullWritable.class, LongWritable.class);
FileOutputFormat.setOutputPath(job, new Path(outputPath));
+ FileOutputFormat.setCompressOutput(job, false);
// prevent to create zero-sized default output
LazyOutputFormat.setOutputFormatClass(job, SequenceFileOutputFormat.class);
+
MultipleOutputsRDD multipleOutputsRDD = MultipleOutputsRDD.rddToMultipleOutputsRDD(outputRDD);
multipleOutputsRDD.saveAsNewAPIHadoopDatasetWithMultipleOutputs(job.getConfiguration());
- logger.info("Map input records={}", recordRDD.count());
+ long recordCount = recordRDD.count();
+ logger.info("Map input records={}", recordCount);
logger.info("HDFS Read: {} HDFS Write", bytesWritten.value());
Map<String, String> counterMap = Maps.newHashMap();
- counterMap.put(ExecutableConstants.SOURCE_RECORDS_COUNT, String.valueOf(recordRDD.count()));
+ counterMap.put(ExecutableConstants.SOURCE_RECORDS_COUNT, String.valueOf(recordCount));
counterMap.put(ExecutableConstants.SOURCE_RECORDS_SIZE, String.valueOf(bytesWritten.value()));
// save counter to hdfs
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
index e332e1d7d7..443eba6021 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
@@ -22,6 +22,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
+import java.util.Collections;
import java.util.List;
import org.apache.commons.cli.Option;
@@ -38,6 +39,8 @@
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.DataModelManager;
+import org.apache.kylin.metadata.model.SegmentRange;
+import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -46,6 +49,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.Lists;
+
/**
* 1. Create new HybridCube
* bin/kylin.sh org.apache.kylin.tool.HybridCubeCLI -action create -name hybrid_name -project project_name -model model_name -cubes cube1,cube2
@@ -202,27 +207,28 @@ private void delete(HybridInstance hybridInstance) throws IOException {
}
private void checkSegmentOffset(List<RealizationEntry> realizationEntries) {
- if (realizationEntries == null || realizationEntries.size() == 0)
- throw new IllegalArgumentException("No realization found");
- if (realizationEntries.size() == 1)
- throw new IllegalArgumentException("Hybrid needs at least 2 cubes");
- long lastOffset = -1;
+ List<SegmentRange> segmentRanges = Lists.newArrayList();
+
for (RealizationEntry entry : realizationEntries) {
if (entry.getType() != RealizationType.CUBE) {
throw new IllegalArgumentException("Wrong realization type: " + entry.getType() + ", only cube supported. ");
}
CubeInstance cubeInstance = cubeManager.getCube(entry.getRealization());
- CubeSegment segment = cubeInstance.getLastSegment();
- if (segment == null)
- continue;
- if (lastOffset == -1) {
- lastOffset = (Long) segment.getSegRange().end.v;
- } else {
- if (lastOffset > (Long) segment.getSegRange().start.v) {
- throw new IllegalArgumentException("Segments has overlap, could not hybrid. Last Segment End: " + lastOffset + ", Next Segment Start: " + segment.getSegRange().start.v);
+ Segments<CubeSegment> segments = cubeInstance.getSegments();
+
+ for (CubeSegment segment : segments) {
+ segmentRanges.add(segment.getSegRange());
+ }
+ }
+
+ if (segmentRanges.size() >= 2) {
+ Collections.sort(segmentRanges);
+
+ for (int i = 0; i < segmentRanges.size() - 1; i++) {
+ if (segmentRanges.get(i).overlaps(segmentRanges.get(i + 1))) {
+ throw new IllegalArgumentException("Segments has overlap, could not hybrid. First Segment Range: [" + segmentRanges.get(i).start.v + "," + segmentRanges.get(i).end.v + "], Second Segment Range: [" + segmentRanges.get(i + 1).start.v + "," + segmentRanges.get(i + 1).end.v + "]");
}
- lastOffset = (Long) segment.getSegRange().end.v;
}
}
}
diff --git a/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
index f4ed595404..99b8319386 100644
--- a/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
+++ b/tool/src/test/java/org/apache/kylin/tool/HybridCubeCLITest.java
@@ -22,6 +22,9 @@
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.storage.hybrid.HybridInstance;
@@ -30,12 +33,17 @@
import org.junit.Assert;
import org.junit.Before;
import org.junit.FixMethodOrder;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.junit.runners.MethodSorters;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class HybridCubeCLITest extends LocalFileMetadataTestCase {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
@Before
public void setUp() throws Exception {
this.createTestMetadata();
@@ -89,4 +97,44 @@ public void test3Delete() throws IOException {
Assert.assertEquals(0, ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).findProjects(RealizationType.HYBRID, "ssb_hybrid").size());
}
+ @Test
+ public void testSegmentOverlap() throws IOException {
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("Segments has overlap");
+
+ HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
+
+ CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
+ CubeInstance cube2 = cubeManager.getCube("ssb_cube2");
+
+ // 2012-01-01,2012-01-03
+ cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
+ // 2012-01-02,2012-01-04
+ cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325462400000L, 1325635200000L));
+
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
+ }
+
+ @Test
+ public void testSegmentNotOverlap() throws IOException {
+ HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
+
+ CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+ CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
+ CubeInstance cube2 = cubeManager.getCube("ssb_cube2");
+
+ // 2012-01-01,2012-01-03
+ cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
+ // 2012-01-03,2012-01-04
+ cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325548800000L, 1325635200000L));
+
+ // not throw exception
+ HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
+ }
+
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services