You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2017/01/05 04:01:41 UTC
[1/2] kylin git commit: KYLIN-2282 Step name Build N-Dimension Cuboid
Data : N-Dimension is inaccurate
Repository: kylin
Updated Branches:
refs/heads/master e6a84c569 -> 809b78309
KYLIN-2282 Step name Build N-Dimension Cuboid Data : N-Dimension is inaccurate
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/809b7830
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/809b7830
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/809b7830
Branch: refs/heads/master
Commit: 809b78309d0b85058aad43e6b863271dfebd1255
Parents: e043d5f
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 5 12:00:12 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 5 12:01:36 2017 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/BatchCubingJobBuilder.java | 27 +++++++++----------
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 28 +++++++++-----------
.../kylin/engine/mr/JobBuilderSupport.java | 11 ++++----
3 files changed, 30 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/809b7830/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
index 6c973eb..8fbecd3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder.java
@@ -63,16 +63,15 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
result.addTask(createBuildDictionaryStep(jobId));
// Phase 3: Build Cube
- RowKeyDesc rowKeyDesc = ((CubeSegment) seg).getCubeDesc().getRowkey();
- final int groupRowkeyColumnsCount = ((CubeSegment) seg).getCubeDesc().getBuildLevel();
+ RowKeyDesc rowKeyDesc = seg.getCubeDesc().getRowkey();
+ final int groupRowkeyColumnsCount = seg.getCubeDesc().getBuildLevel();
final int totalRowkeyColumnsCount = rowKeyDesc.getRowKeyColumns().length;
- final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount);
+ final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, groupRowkeyColumnsCount);
// base cuboid step
- result.addTask(createBaseCuboidStep(cuboidOutputTempPath, jobId));
+ result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId));
// n dim cuboid steps
- for (int i = 1; i <= groupRowkeyColumnsCount; i++) {
- int dimNum = totalRowkeyColumnsCount - i;
- result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath, dimNum, totalRowkeyColumnsCount));
+ for (int i = 1; i < groupRowkeyColumnsCount; i++) {
+ result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath[i - 1], cuboidOutputTempPath[i], i));
}
outputSide.addStepPhase3_BuildCube(result, cuboidRootPath);
@@ -84,7 +83,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
return result;
}
- private MapReduceExecutable createBaseCuboidStep(String[] cuboidOutputTempPath, String jobId) {
+ private MapReduceExecutable createBaseCuboidStep(String cuboidOutputPath, String jobId) {
// base cuboid job
MapReduceExecutable baseCuboidStep = new MapReduceExecutable();
@@ -96,7 +95,7 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, "FLAT_TABLE"); // marks flat table input
- appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[0]);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputPath);
appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Base_Cuboid_Builder_" + seg.getRealization().getName());
appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "0");
appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
@@ -107,20 +106,20 @@ public class BatchCubingJobBuilder extends JobBuilderSupport {
return baseCuboidStep;
}
- private MapReduceExecutable createNDimensionCuboidStep(String[] cuboidOutputTempPath, int dimNum, int totalRowkeyColumnCount) {
+ private MapReduceExecutable createNDimensionCuboidStep(String parentPath, String outputPath, int level) {
// ND cuboid job
MapReduceExecutable ndCuboidStep = new MapReduceExecutable();
- ndCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_N_D_CUBOID + " : " + dimNum + "-Dimension");
+ ndCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_N_D_CUBOID + " : level " + level);
StringBuilder cmd = new StringBuilder();
appendMapReduceParameters(cmd);
appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
- appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum - 1]);
- appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum]);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, parentPath);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath);
appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_ND-Cuboid_Builder_" + seg.getRealization().getName() + "_Step");
- appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "" + (totalRowkeyColumnCount - dimNum));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "" + level);
ndCuboidStep.setMapReduceParams(cmd.toString());
ndCuboidStep.setMapReduceJobClass(NDCuboidJob.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/809b7830/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
index 966bb1b..20540a6 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/BatchCubingJobBuilder2.java
@@ -19,7 +19,6 @@
package org.apache.kylin.engine.mr;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.RowKeyDesc;
import org.apache.kylin.engine.mr.IMRInput.IMRBatchCubingInputSide;
import org.apache.kylin.engine.mr.IMROutput2.IMRBatchCubingOutputSide2;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
@@ -77,16 +76,13 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
}
private void addLayerCubingSteps(final CubingJob result, final String jobId, final String cuboidRootPath) {
- RowKeyDesc rowKeyDesc = ((CubeSegment) seg).getCubeDesc().getRowkey();
- final int groupRowkeyColumnsCount = ((CubeSegment) seg).getCubeDesc().getBuildLevel();
- final int totalRowkeyColumnsCount = rowKeyDesc.getRowKeyColumns().length;
- final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, totalRowkeyColumnsCount, groupRowkeyColumnsCount);
+ final int maxLevel = seg.getCubeDesc().getBuildLevel();
+ final String[] cuboidOutputTempPath = getCuboidOutputPaths(cuboidRootPath, maxLevel);
// base cuboid step
- result.addTask(createBaseCuboidStep(cuboidOutputTempPath, jobId));
+ result.addTask(createBaseCuboidStep(cuboidOutputTempPath[0], jobId));
// n dim cuboid steps
- for (int i = 1; i < groupRowkeyColumnsCount; i++) {
- int dimNum = totalRowkeyColumnsCount - i;
- result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath, dimNum, totalRowkeyColumnsCount, jobId));
+ for (int i = 1; i < maxLevel; i++) {
+ result.addTask(createNDimensionCuboidStep(cuboidOutputTempPath[i - 1], cuboidOutputTempPath[i], i, jobId));
}
}
@@ -125,7 +121,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
return InMemCuboidJob.class;
}
- private MapReduceExecutable createBaseCuboidStep(String[] cuboidOutputTempPath, String jobId) {
+ private MapReduceExecutable createBaseCuboidStep(String cuboidOutputPath, String jobId) {
// base cuboid job
MapReduceExecutable baseCuboidStep = new MapReduceExecutable();
@@ -137,7 +133,7 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, "FLAT_TABLE"); // marks flat table input
- appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[0]);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputPath);
appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Base_Cuboid_Builder_" + seg.getRealization().getName());
appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "0");
appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
@@ -152,20 +148,20 @@ public class BatchCubingJobBuilder2 extends JobBuilderSupport {
return BaseCuboidJob.class;
}
- private MapReduceExecutable createNDimensionCuboidStep(String[] cuboidOutputTempPath, int dimNum, int totalRowkeyColumnCount, String jobId) {
+ private MapReduceExecutable createNDimensionCuboidStep(String parentPath, String outputPath, int level, String jobId) {
// ND cuboid job
MapReduceExecutable ndCuboidStep = new MapReduceExecutable();
- ndCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_N_D_CUBOID + " : " + dimNum + "-Dimension");
+ ndCuboidStep.setName(ExecutableConstants.STEP_NAME_BUILD_N_D_CUBOID + " : level " + level);
StringBuilder cmd = new StringBuilder();
appendMapReduceParameters(cmd);
appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
- appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum - 1]);
- appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidOutputTempPath[totalRowkeyColumnCount - dimNum]);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, parentPath);
+ appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, outputPath);
appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_ND-Cuboid_Builder_" + seg.getRealization().getName() + "_Step");
- appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "" + (totalRowkeyColumnCount - dimNum));
+ appendExecCmdParameters(cmd, BatchConstants.ARG_LEVEL, "" + level);
appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
ndCuboidStep.setMapReduceParams(cmd.toString());
http://git-wip-us.apache.org/repos/asf/kylin/blob/809b7830/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 5f5814b..23fce2b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -192,14 +192,13 @@ public class JobBuilderSupport {
return buf.append(" -").append(paraName).append(" ").append(paraValue);
}
- public String[] getCuboidOutputPaths(String cuboidRootPath, int totalRowkeyColumnCount, int groupRowkeyColumnsCount) {
- String[] paths = new String[groupRowkeyColumnsCount + 1];
- for (int i = 0; i <= groupRowkeyColumnsCount; i++) {
- int dimNum = totalRowkeyColumnCount - i;
- if (dimNum == totalRowkeyColumnCount) {
+ public String[] getCuboidOutputPaths(String cuboidRootPath, int levels) {
+ String[] paths = new String[levels];
+ for (int i = 0; i < levels; i++) {
+ if (i == 0) {
paths[i] = cuboidRootPath + "base_cuboid";
} else {
- paths[i] = cuboidRootPath + dimNum + "d_cuboid";
+ paths[i] = cuboidRootPath + "level_" + i + "_cuboid";
}
}
return paths;
[2/2] kylin git commit: KYLIN-2342 When NoClassDefFoundError occurred
in building cube, no error in kylin.log
Posted by sh...@apache.org.
KYLIN-2342 When NoClassDefFoundError occurred in building cube, no error in kylin.log
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e043d5fa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e043d5fa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e043d5fa
Branch: refs/heads/master
Commit: e043d5fad8ccf420b1c0e744ee87a237cd7babd7
Parents: e6a84c5
Author: shaofengshi <sh...@apache.org>
Authored: Thu Jan 5 10:54:33 2017 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Jan 5 12:01:36 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller/CubeController.java | 4 ++--
.../org/apache/kylin/rest/controller/TableController.java | 10 ++++++----
.../org/apache/kylin/rest/init/InitialTaskManager.java | 2 +-
3 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e043d5fa/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 978f477..a9c8321 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -293,7 +293,7 @@ public class CubeController extends BasicController {
}
return jobService.submitJob(cube, startTime, endTime, startOffset, endOffset, //
sourcePartitionOffsetStart, sourcePartitionOffsetEnd, CubeBuildTypeEnum.valueOf(buildType), force, submitter);
- } catch (Exception e) {
+ } catch (Throwable e) {
logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e.getLocalizedMessage());
}
@@ -656,7 +656,7 @@ public class CubeController extends BasicController {
cubeService.getCubeDescManager().updateCubeDesc(desc);
response.setProperty("result", "success");
response.setProperty("offsets", startOffsets.toString());
- } catch (Exception e) {
+ } catch (Throwable e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e043d5fa/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index 1ab6ba6..ac4ee2f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -112,7 +112,7 @@ public class TableController extends BasicController {
if (request.isCalculate()) {
tableService.calculateCardinalityIfNotPresent(loaded, submitter);
}
- } catch (Exception e) {
+ } catch (Throwable e) {
logger.error("Failed to load Hive Table", e);
throw new InternalErrorException(e.getLocalizedMessage());
}
@@ -133,7 +133,7 @@ public class TableController extends BasicController {
unLoadFail.add(tableName);
}
}
- } catch (Exception e) {
+ } catch (Throwable e) {
logger.error("Failed to unload Hive Table", e);
throw new InternalErrorException(e.getLocalizedMessage());
}
@@ -175,7 +175,8 @@ public class TableController extends BasicController {
private List<String> showHiveDatabases() throws IOException {
try {
return tableService.getHiveDbNames();
- } catch (Exception e) {
+ } catch (Throwable e) {
+ logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e.getLocalizedMessage());
}
}
@@ -191,7 +192,8 @@ public class TableController extends BasicController {
private List<String> showHiveTables(@PathVariable String database) throws IOException {
try {
return tableService.getHiveTableNames(database);
- } catch (Exception e) {
+ } catch (Throwable e) {
+ logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e.getLocalizedMessage());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e043d5fa/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
index 5cf48cb..8bba674 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
@@ -52,7 +52,7 @@ public class InitialTaskManager implements InitializingBean {
InitialTask task = (InitialTask) Class.forName(taskClass).newInstance();
logger.info("Running task: " + taskClass);
task.execute();
- } catch (Exception e) {
+ } catch (Throwable e) {
logger.error("Initial task failed: " + taskClass, e);
}
}