You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/02/05 09:40:27 UTC
kylin git commit: KYLIN-1402 StringIndexOutOfBoundsException in Kylin
Hive Column Cardinality Job
Repository: kylin
Updated Branches:
refs/heads/2.x-staging 5b636df32 -> b180e2ec6
KYLIN-1402 StringIndexOutOfBoundsException in Kylin Hive Column Cardinality Job
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b180e2ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b180e2ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b180e2ec
Branch: refs/heads/2.x-staging
Commit: b180e2ec69b3f44c28ce709995acf04f65027243
Parents: 5b636df
Author: shaofengshi <sh...@apache.org>
Authored: Fri Feb 5 16:40:07 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Feb 5 16:40:07 2016 +0800
----------------------------------------------------------------------
.../HiveColumnCardinalityUpdateJob.java | 36 ++++++++++++--------
1 file changed, 22 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b180e2ec/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
index d088bfa..9a4d8de 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
@@ -18,14 +18,6 @@
package org.apache.kylin.source.hive.cardinality;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
@@ -41,11 +33,21 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
/**
* This job will update save the cardinality result into Kylin table metadata store.
- * @author shaoshi
*
+ * @author shaoshi
*/
public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
public static final String JOB_TITLE = "Kylin Hive Column Cardinality Update Job";
@@ -55,6 +57,8 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
private String table;
+ private static final Logger logger = LoggerFactory.getLogger(HiveColumnCardinalityUpdateJob.class);
+
public HiveColumnCardinalityUpdateJob() {
}
@@ -110,11 +114,15 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
cardi.append(",");
}
String scardi = cardi.toString();
- scardi = scardi.substring(0, scardi.length() - 1);
- MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- Map<String, String> tableExd = metaMgr.getTableDescExd(tableName);
- tableExd.put(MetadataConstants.TABLE_EXD_CARDINALITY, scardi);
- metaMgr.saveTableExd(tableName.toUpperCase(), tableExd);
+ if (scardi.length() > 0) {
+ scardi = scardi.substring(0, scardi.length() - 1);
+ MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Map<String, String> tableExd = metaMgr.getTableDescExd(tableName);
+ tableExd.put(MetadataConstants.TABLE_EXD_CARDINALITY, scardi);
+ metaMgr.saveTableExd(tableName.toUpperCase(), tableExd);
+ } else {
+ throw new IllegalArgumentException("No cardinality data is collected for table " + tableName);
+ }
}
private static List<String> readLines(Path location, Configuration conf) throws Exception {