You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/02/21 12:52:16 UTC
[21/52] [abbrv] kylin git commit: KYLIN-1402
StringIndexOutOfBoundsException in Kylin Hive Column Cardinality Job
KYLIN-1402 StringIndexOutOfBoundsException in Kylin Hive Column Cardinality Job
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/85d55f4f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/85d55f4f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/85d55f4f
Branch: refs/heads/1.x-HBase1.1.3
Commit: 85d55f4f8e4de440c7764cd1820917ddbef3836b
Parents: 474dbfa
Author: shaofengshi <sh...@apache.org>
Authored: Fri Feb 5 16:35:25 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Feb 5 16:35:25 2016 +0800
----------------------------------------------------------------------
.../HiveColumnCardinalityUpdateJob.java | 44 +++++++++++---------
1 file changed, 25 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/85d55f4f/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
index 115753e..cad3c0b 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cardinality/HiveColumnCardinalityUpdateJob.java
@@ -18,14 +18,6 @@
package org.apache.kylin.job.hadoop.cardinality;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
@@ -41,11 +33,21 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.job.hadoop.AbstractHadoopJob;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
/**
* This job will update save the cardinality result into Kylin table metadata store.
- * @author shaoshi
*
+ * @author shaoshi
*/
public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
public static final String JOB_TITLE = "Kylin Hive Column Cardinality Update Job";
@@ -53,6 +55,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
@SuppressWarnings("static-access")
protected static final Option OPTION_TABLE = OptionBuilder.withArgName("table name").hasArg().isRequired(true).withDescription("The hive table name").create("table");
+ private static final Logger logger = LoggerFactory.getLog(HiveColumnCardinalityUpdateJob.class);
private String table;
public HiveColumnCardinalityUpdateJob() {
@@ -73,7 +76,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
this.table = getOptionValue(OPTION_TABLE).toUpperCase();
// start job
String jobName = JOB_TITLE + getOptionsAsString();
- System.out.println("Starting: " + jobName);
+ logger.info("Starting: " + jobName);
Configuration conf = getConf();
Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH));
@@ -91,30 +94,33 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
try {
columns = readLines(new Path(outPath), config);
} catch (Exception e) {
- e.printStackTrace();
- System.out.println("Failed to resolve cardinality for " + tableName + " from " + outPath);
+ logger.error("Failed to resolve cardinality for " + tableName + " from " + outPath, e);
return;
}
StringBuffer cardi = new StringBuffer();
Iterator<String> it = columns.iterator();
while (it.hasNext()) {
- String string = (String) it.next();
+ String string = it.next();
String[] ss = StringUtils.split(string, "\t");
if (ss.length != 2) {
- System.out.println("The hadoop cardinality value is not valid " + string);
+ logger.info("The hadoop cardinality value is not valid " + string);
continue;
}
cardi.append(ss[1]);
cardi.append(",");
}
String scardi = cardi.toString();
- scardi = scardi.substring(0, scardi.length() - 1);
- MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- Map<String, String> tableExd = metaMgr.getTableDescExd(tableName);
- tableExd.put(MetadataConstants.TABLE_EXD_CARDINALITY, scardi);
- metaMgr.saveTableExd(tableName.toUpperCase(), tableExd);
+ if (scardi.length() > 0) {
+ scardi = scardi.substring(0, scardi.length() - 1);
+ MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Map<String, String> tableExd = metaMgr.getTableDescExd(tableName);
+ tableExd.put(MetadataConstants.TABLE_EXD_CARDINALITY, scardi);
+ metaMgr.saveTableExd(tableName.toUpperCase(), tableExd);
+ } else {
+ throw new IllegalArgumentException("No cardinality data is collected for table " + tableName);
+ }
}
private static List<String> readLines(Path location, Configuration conf) throws Exception {