You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/05/11 13:29:58 UTC
[kylin] branch KYLIN-3368 created (now 11a7cd5)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch KYLIN-3368
in repository https://gitbox.apache.org/repos/asf/kylin.git.
at 11a7cd5 KYLIN-3368 Move Spark cubing metadata dump to job folder
This branch includes the following new commits:
new 11a7cd5 KYLIN-3368 Move Spark cubing metadata dump to job folder
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 01/01: KYLIN-3368 Move Spark cubing metadata dump to job
folder
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch KYLIN-3368
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 11a7cd506887af3d130d4470c7eee5bfca4aefd9
Author: shaofengshi <sh...@apache.org>
AuthorDate: Fri May 11 21:29:40 2018 +0800
KYLIN-3368 Move Spark cubing metadata dump to job folder
---
.../java/org/apache/kylin/engine/mr/JobBuilderSupport.java | 5 +++++
.../kylin/engine/spark/SparkBatchCubingJobBuilder2.java | 12 ++++--------
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 8228f87..8a420df 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -261,4 +261,9 @@ public class JobBuilderSupport {
public static String getInMemCuboidPath(String cuboidRootPath) {
return cuboidRootPath + PathNameCuboidInMem;
}
+
+ public String getDumpMetadataPath(String jobId) {
+ return getRealizationRootPath(jobId) + "/metadata";
+ }
+
}
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
index 7d76ce4..57d4fb0 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java
@@ -67,7 +67,7 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
return "";
}
- public static void configureSparkJob(final CubeSegment seg, final SparkExecutable sparkExecutable,
+ public void configureSparkJob(final CubeSegment seg, final SparkExecutable sparkExecutable,
final String jobId, final String cuboidRootPath) {
IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(seg);
sparkExecutable.setParam(SparkCubingByLayer.OPTION_CUBE_NAME.getOpt(), seg.getRealization().getName());
@@ -75,15 +75,12 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
sparkExecutable.setParam(SparkCubingByLayer.OPTION_INPUT_TABLE.getOpt(),
seg.getConfig().getHiveDatabaseForIntermediateTable() + "." + flatTableDesc.getTableName());
sparkExecutable.setParam(SparkCubingByLayer.OPTION_META_URL.getOpt(),
- getSegmentMetadataUrl(seg.getConfig(), seg.getUuid()));
+ getSegmentMetadataUrl(seg.getConfig(), jobId));
sparkExecutable.setParam(SparkCubingByLayer.OPTION_OUTPUT_PATH.getOpt(), cuboidRootPath);
sparkExecutable.setJobId(jobId);
StringBuilder jars = new StringBuilder();
- StringUtil.appendWithSeparator(jars, findJar("org.htrace.HTraceConfiguration", null)); // htrace-core.jar
- StringUtil.appendWithSeparator(jars, findJar("org.apache.htrace.Trace", null)); // htrace-core.jar
- StringUtil.appendWithSeparator(jars, findJar("org.cloudera.htrace.HTraceConfiguration", null)); // htrace-core.jar
StringUtil.appendWithSeparator(jars, findJar("com.yammer.metrics.core.Gauge", null)); // metrics-core.jar
StringUtil.appendWithSeparator(jars, findJar("com.google.common.collect.Maps", "guava")); //guava.jar
@@ -92,10 +89,9 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 {
sparkExecutable.setName(ExecutableConstants.STEP_NAME_BUILD_SPARK_CUBE);
}
- private static String getSegmentMetadataUrl(KylinConfig kylinConfig, String segmentID) {
+ private String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) {
Map<String, String> param = new HashMap<>();
- param.put("path", kylinConfig.getHdfsWorkingDirectory() + "metadata/" + segmentID);
+ param.put("path", getDumpMetadataPath(jobId));
return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString();
-// return kylinConfig.getHdfsWorkingDirectory() + "metadata/" + segmentID + "@hdfs";
}
}
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.