You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/09/12 01:59:45 UTC
[kylin] branch master updated (91fb3c1 -> 3f7b42a)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 91fb3c1 KYLIN-3547 Add DataTpyeOrder boolean
new 7cba779 KYLIN-3550 kylin.source.hive.flat-table-field-delimiter extra \
new 9e5856a KYLIN-3554 Kylin move to next step on Spark job resumed
new 0f7d3f8 minor, update front end error message
new 3f7b42a KYLIN-3557 PreparedStatement not closed in JDBCResourceDAO
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/kylin/common/persistence/JDBCResourceDAO.java | 12 +++++++++---
.../src/main/java/org/apache/kylin/job/JoinedFlatTable.java | 2 +-
.../java/org/apache/kylin/engine/spark/SparkExecutable.java | 13 +++++++++----
webapp/app/js/controllers/cube.js | 2 +-
4 files changed, 20 insertions(+), 9 deletions(-)
[kylin] 04/04: KYLIN-3557 PreparedStatement not closed in
JDBCResourceDAO
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3f7b42acd1ef2b99f054cc958b310d1cd7b6784c
Author: shaofengshi <sh...@apache.org>
AuthorDate: Wed Sep 12 09:59:36 2018 +0800
KYLIN-3557 PreparedStatement not closed in JDBCResourceDAO
---
.../org/apache/kylin/common/persistence/JDBCResourceDAO.java | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/JDBCResourceDAO.java b/core-common/src/main/java/org/apache/kylin/common/persistence/JDBCResourceDAO.java
index 9bf6c24..a226af6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/JDBCResourceDAO.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/JDBCResourceDAO.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.DBUtils;
import org.apache.kylin.common.util.HadoopUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -441,10 +442,15 @@ public class JDBCResourceDAO {
private boolean checkTableExists(final String tableName, final Connection connection) throws SQLException {
final PreparedStatement ps = connection.prepareStatement(getCheckTableExistsSql(tableName));
final ResultSet rs = ps.executeQuery();
- while (rs.next()) {
- if (tableName.equals(rs.getString(1))) {
- return true;
+ try {
+ while (rs.next()) {
+ if (tableName.equals(rs.getString(1))) {
+ return true;
+ }
}
+ } finally {
+ DBUtils.closeQuietly(rs);
+ DBUtils.closeQuietly(ps);
}
return false;
[kylin] 01/04: KYLIN-3550
kylin.source.hive.flat-table-field-delimiter extra \
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 7cba77993e3372ce8764a9a2e8bcd5bb2e6a18cf
Author: shaofengshi <sh...@apache.org>
AuthorDate: Tue Sep 11 22:24:33 2018 +0800
KYLIN-3550 kylin.source.hive.flat-table-field-delimiter extra \
---
core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 9a0879f..ff48244 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -90,7 +90,7 @@ public class JoinedFlatTable {
}
ddl.append(")" + "\n");
if (TEXTFILE.equals(storageFormat)) {
- ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\" + fieldDelimiter + "'\n");
+ ddl.append("ROW FORMAT DELIMITED FIELDS TERMINATED BY '" + fieldDelimiter + "'\n");
}
ddl.append("STORED AS " + storageFormat + "\n");
ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + "';").append("\n");
[kylin] 02/04: KYLIN-3554 Kylin move to next step on Spark job
resumed
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9e5856ae15eecd08658026963641f3a586a17d96
Author: shaofengshi <sh...@apache.org>
AuthorDate: Tue Sep 11 22:25:46 2018 +0800
KYLIN-3554 Kylin move to next step on Spark job resumed
---
.../java/org/apache/kylin/engine/spark/SparkExecutable.java | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
index f4f06be..e4f8c43 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java
@@ -190,12 +190,14 @@ public class SparkExecutable extends AbstractExecutable {
}
+ @SuppressWarnings("checkstyle:methodlength")
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
ExecutableManager mgr = getManager();
Map<String, String> extra = mgr.getOutput(getId()).getExtra();
- if (extra.containsKey(ExecutableConstants.SPARK_JOB_ID)) {
- return onResumed(extra.get(ExecutableConstants.SPARK_JOB_ID), mgr);
+ String sparkJobId = extra.get(ExecutableConstants.SPARK_JOB_ID);
+ if (!StringUtils.isEmpty(sparkJobId)) {
+ return onResumed(sparkJobId, mgr);
} else {
String cubeName = this.getParam(SparkCubingByLayer.OPTION_CUBE_NAME.getOpt());
CubeInstance cube = CubeManager.getInstance(context.getConfig()).getCube(cubeName);
@@ -326,14 +328,17 @@ public class SparkExecutable extends AbstractExecutable {
Map<String, String> counterMap = HadoopUtil.readFromSequenceFile(counterOutput);
joblogInfo.putAll(counterMap);
} else {
- logger.warn("Spark counter output path not exists");
+ logger.warn("Spark counter output path not exists: " + counterOutput);
}
}
readCounters(joblogInfo);
getManager().addJobInfo(getId(), joblogInfo);
return new ExecuteResult(ExecuteResult.State.SUCCEED, patternedLogger.getBufferedLog());
}
-
+ // clear SPARK_JOB_ID on job failure.
+ extra = mgr.getOutput(getId()).getExtra();
+ extra.put(ExecutableConstants.SPARK_JOB_ID, "");
+ getManager().addJobInfo(getId(), extra);
return new ExecuteResult(ExecuteResult.State.ERROR, result != null ? result.getSecond() : "");
} catch (Exception e) {
logger.error("error run spark job:", e);
[kylin] 03/04: minor, update front end error message
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0f7d3f8a7c19712b65269bdc4af8f9c485faa326
Author: shaofengshi <sh...@apache.org>
AuthorDate: Tue Sep 11 22:26:00 2018 +0800
minor, update front end error message
---
webapp/app/js/controllers/cube.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/webapp/app/js/controllers/cube.js b/webapp/app/js/controllers/cube.js
index aff3226..60d141b 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -169,7 +169,7 @@ KylinApp.controller('CubeCtrl', function ($scope, $rootScope, AccessService, Mes
}
}, function(e) {
loadingRequest.hide();
- SweetAlert.swal('Oops...', 'Failed to get recommend cuboid.', 'error');
+ SweetAlert.swal('Oops...', 'Failed to get recommend Cuboid; Check whether you have enabled System Cube.', 'error');
console.error('recommend cuboid error', e.data);
});
} else {