You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/12/14 06:59:22 UTC
[1/3] kylin git commit: KYLIN-2996 DeployCoprocessorCLI Log failed
tables info
Repository: kylin
Updated Branches:
refs/heads/master b5a02430b -> 280f6738b
KYLIN-2996 DeployCoprocessorCLI Log failed tables info
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a79035a6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a79035a6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a79035a6
Branch: refs/heads/master
Commit: a79035a63c6928eee6d556bd15bc8e5c50880ae8
Parents: b5a0243
Author: kangkaisen <ka...@meituan.com>
Authored: Wed Sep 6 16:11:48 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 14 14:58:00 2017 +0800
----------------------------------------------------------------------
.../hbase/util/DeployCoprocessorCLI.java | 27 ++++++++++++++------
1 file changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a79035a6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index c437e66..56d5497 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinVersion;
import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -122,7 +123,7 @@ public class DeployCoprocessorCLI {
Path hdfsCoprocessorJar = uploadCoprocessorJar(localCoprocessorJar, fileSystem, oldJarPaths);
logger.info("New coprocessor jar: " + hdfsCoprocessorJar);
- List<String> processedTables = resetCoprocessorOnHTables(hbaseAdmin, hdfsCoprocessorJar, tableNames);
+ Pair<List<String>, List<String>> results = resetCoprocessorOnHTables(hbaseAdmin, hdfsCoprocessorJar, tableNames);
// Don't remove old jars, missing coprocessor jar will fail hbase
// removeOldJars(oldJarPaths, fileSystem);
@@ -130,8 +131,10 @@ public class DeployCoprocessorCLI {
hbaseAdmin.close();
logger.info("Processed time: " + (System.currentTimeMillis() - start));
- logger.info("Processed tables count: " + processedTables.size());
- logger.info("Processed tables: " + processedTables);
+ logger.info("Processed tables count: " + results.getFirst().size());
+ logger.info("Processed tables: " + results.getFirst());
+ logger.error("Failed tables count: " + results.getSecond().size());
+ logger.error("Failed tables : " + results.getSecond());
logger.info("Active coprocessor jar: " + hdfsCoprocessorJar);
}
@@ -157,7 +160,7 @@ public class DeployCoprocessorCLI {
ProjectInstance projectInstance = projectManager.getProject(p);
List<RealizationEntry> cubeList = projectInstance.getRealizationEntries(RealizationType.CUBE);
- for (RealizationEntry cube: cubeList) {
+ for (RealizationEntry cube : cubeList) {
CubeInstance cubeInstance = cubeManager.getCube(cube.getRealization());
for (CubeSegment segment : cubeInstance.getSegments()) {
String tableName = segment.getStorageLocationIdentifier();
@@ -279,13 +282,15 @@ public class DeployCoprocessorCLI {
return true;
}
- private static List<String> resetCoprocessorOnHTables(final Admin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
+ private static Pair<List<String>, List<String>> resetCoprocessorOnHTables(final Admin hbaseAdmin, final Path hdfsCoprocessorJar, List<String> tableNames) throws IOException {
List<String> processedTables = Collections.synchronizedList(new ArrayList<String>());
+ List<String> failedTables = Collections.synchronizedList(new ArrayList<String>());
+
ExecutorService coprocessorPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
CountDownLatch countDownLatch = new CountDownLatch(tableNames.size());
for (final String tableName : tableNames) {
- coprocessorPool.execute(new ResetCoprocessorWorker(countDownLatch, hbaseAdmin, hdfsCoprocessorJar, tableName, processedTables));
+ coprocessorPool.execute(new ResetCoprocessorWorker(countDownLatch, hbaseAdmin, hdfsCoprocessorJar, tableName, processedTables, failedTables));
}
try {
@@ -295,7 +300,7 @@ public class DeployCoprocessorCLI {
}
coprocessorPool.shutdown();
- return processedTables;
+ return new Pair<>(processedTables, failedTables);
}
private static class ResetCoprocessorWorker implements Runnable {
@@ -304,13 +309,16 @@ public class DeployCoprocessorCLI {
private final Path hdfsCoprocessorJar;
private final String tableName;
private final List<String> processedTables;
+ private final List<String> failedTables;
+
+ public ResetCoprocessorWorker(CountDownLatch countDownLatch, Admin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables, List<String> failedTables) {
- public ResetCoprocessorWorker(CountDownLatch countDownLatch, Admin hbaseAdmin, Path hdfsCoprocessorJar, String tableName, List<String> processedTables) {
this.countDownLatch = countDownLatch;
this.hbaseAdmin = hbaseAdmin;
this.hdfsCoprocessorJar = hdfsCoprocessorJar;
this.tableName = tableName;
this.processedTables = processedTables;
+ this.failedTables = failedTables;
}
@Override
@@ -319,8 +327,11 @@ public class DeployCoprocessorCLI {
boolean isProcessed = resetCoprocessor(tableName, hbaseAdmin, hdfsCoprocessorJar);
if (isProcessed) {
processedTables.add(tableName);
+ } else {
+ failedTables.add(tableName);
}
} catch (Exception ex) {
+ failedTables.add(tableName);
logger.error("Error processing " + tableName, ex);
} finally {
countDownLatch.countDown();
[2/3] kylin git commit: KYLIN-2993 Add special mr config for base
cuboid step
Posted by li...@apache.org.
KYLIN-2993 Add special mr config for base cuboid step
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad3dd3f6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad3dd3f6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad3dd3f6
Branch: refs/heads/master
Commit: ad3dd3f63d399640142bd2c97669080bc3e37560
Parents: a79035a
Author: kangkaisen <ka...@meituan.com>
Authored: Mon Sep 4 10:29:00 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 14 14:58:17 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../kylin/storage/hbase/steps/HBaseMROutput2Transition.java | 9 +++++++++
2 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad3dd3f6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d59c98d..524b1d4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1003,6 +1003,10 @@ abstract public class KylinConfigBase implements Serializable {
return getPropertiesByPrefix("kylin.engine.mr.uhc-config-override.");
}
+ public Map<String, String> getBaseCuboidMRConfigOverride() {
+ return getPropertiesByPrefix("kylin.engine.mr.base-cuboid-config-override.");
+ }
+
public Map<String, String> getSparkConfigOverride() {
return getPropertiesByPrefix("kylin.engine.spark-conf.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad3dd3f6/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
index db3f7f4..8817cb2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2Transition.java
@@ -19,6 +19,7 @@
package org.apache.kylin.storage.hbase.steps;
import java.util.List;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -99,6 +100,14 @@ public class HBaseMROutput2Transition implements IMROutput2 {
int level) throws Exception {
int reducerNum = 1;
Class mapperClass = job.getMapperClass();
+
+ //allow user specially set config for base cuboid step
+ if (mapperClass == HiveToBaseCuboidMapper.class) {
+ for (Map.Entry<String, String> entry : segment.getConfig().getBaseCuboidMRConfigOverride().entrySet()) {
+ job.getConfiguration().set(entry.getKey(), entry.getValue());
+ }
+ }
+
if (mapperClass == HiveToBaseCuboidMapper.class || mapperClass == NDCuboidMapper.class) {
reducerNum = MapReduceUtil.getLayeredCubingReduceTaskNum(segment, cuboidScheduler,
AbstractHadoopJob.getTotalMapInputMB(job), level);
[3/3] kylin git commit: KYLIN-2994 Handle NPE when load dict in
DictionaryManager
Posted by li...@apache.org.
KYLIN-2994 Handle NPE when load dict in DictionaryManager
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/280f6738
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/280f6738
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/280f6738
Branch: refs/heads/master
Commit: 280f6738b765aa32dbcdc50951777a8d30e6b489
Parents: ad3dd3f
Author: kangkaisen <ka...@meituan.com>
Authored: Mon Sep 11 14:59:15 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Thu Dec 14 14:58:21 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/dict/DictionaryManager.java | 4 ++++
.../org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java | 5 +++++
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/280f6738/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index e97899c..232f0ea 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -115,6 +115,10 @@ public class DictionaryManager {
public DictionaryInfo getDictionaryInfo(final String resourcePath) throws IOException {
try {
+ //when all the value for this column is NULL, the resourcePath will be NULL
+ if (resourcePath == null) {
+ return NONE_INDICATOR;
+ }
DictionaryInfo result = dictCache.get(resourcePath);
if (result == NONE_INDICATOR) {
return null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/280f6738/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index 2c85a28..255315a 100755
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -163,6 +163,11 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
DictionaryManager dictMgr = DictionaryManager.getInstance(config);
Dictionary<String> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(col));
+ // handle the dict of all merged segments is null
+ if (mergedDict == null) {
+ continue;
+ }
+
Dictionary<String> sourceDict;
// handle the column that all records is null
if (sourceCubeSegment.getDictionary(col) == null) {