You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/02/04 00:51:04 UTC
[01/50] [abbrv] kylin git commit: KYLIN-3175,
update streaming segment's tsrange after merge [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/sync c875c5f3e -> 04b322073 (forced update)
KYLIN-3175, update streaming segment's tsrange after merge
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/16236ef3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/16236ef3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/16236ef3
Branch: refs/heads/sync
Commit: 16236ef30cde20e474740fdee693ce26dbd88155
Parents: 35979ff
Author: Cheng Wang <ch...@kyligence.io>
Authored: Wed Jan 17 15:55:16 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeManager.java | 33 ++++++++++++--------
.../kylin/provision/BuildCubeWithStream.java | 2 +-
2 files changed, 21 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/16236ef3/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index df1d95e..8bdb5aa 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -276,17 +276,17 @@ public class CubeManager implements IRealizationProvider {
return updateCube(update);
}
}
-
+
public CubeInstance updateCubeSegStatus(CubeSegment seg, SegmentStatusEnum status) throws IOException {
try (AutoLock lock = cubeMapLock.lockForWrite()) {
CubeInstance cube = seg.getCubeInstance().latestCopyForWrite();
seg = cube.getSegmentById(seg.getUuid());
-
+
CubeUpdate update = new CubeUpdate(cube);
seg.setStatus(status);
update.setToUpdateSegs(seg);
return updateCube(update);
- }
+ }
}
private CubeInstance updateCubeWithRetry(CubeUpdate update, int retry) throws IOException {
@@ -523,7 +523,7 @@ public class CubeManager implements IRealizationProvider {
return segAssist.optimizeSegments(cube, cuboidsRecommend);
}
}
-
+
public CubeSegment mergeSegments(CubeInstance cube, TSRange tsRange, SegmentRange segRange, boolean force)
throws IOException {
try (AutoLock lock = cubeMapLock.lockForWrite()) {
@@ -547,9 +547,9 @@ public class CubeManager implements IRealizationProvider {
CubeSegment... optimizedSegments) throws IOException {
try (AutoLock lock = cubeMapLock.lockForWrite()) {
segAssist.promoteCheckpointOptimizeSegments(cube, recommendCuboids, optimizedSegments);
- }
+ }
}
-
+
public List<CubeSegment> calculateHoles(String cubeName) {
return segAssist.calculateHoles(cubeName);
}
@@ -683,8 +683,13 @@ public class CubeManager implements IRealizationProvider {
tsRange = null;
Preconditions.checkArgument(segRange != null);
} else {
- if(tsRange == null) {
- tsRange = new TSRange((Long)segRange.start.v, (Long)segRange.end.v);
+ /**In case of non-streaming segment,
+ * tsRange is the same as segRange,
+ * either could fulfill the merge job,
+ * so it needs to convert segRange to tsRange if tsRange is null.
+ **/
+ if (tsRange == null) {
+ tsRange = new TSRange((Long) segRange.start.v, (Long) segRange.end.v);
}
segRange = null;
}
@@ -832,10 +837,11 @@ public class CubeManager implements IRealizationProvider {
updateCube(update);
}
- public void promoteNewlyOptimizeSegments(CubeInstance cube, CubeSegment... optimizedSegments) throws IOException {
+ public void promoteNewlyOptimizeSegments(CubeInstance cube, CubeSegment... optimizedSegments)
+ throws IOException {
CubeInstance cubeCopy = cube.latestCopyForWrite();
CubeSegment[] segCopy = cube.regetSegments(optimizedSegments);
-
+
for (CubeSegment seg : segCopy) {
seg.setStatus(SegmentStatusEnum.READY_PENDING);
}
@@ -849,12 +855,12 @@ public class CubeManager implements IRealizationProvider {
CubeSegment... optimizedSegments) throws IOException {
CubeInstance cubeCopy = cube.latestCopyForWrite();
CubeSegment[] optSegCopy = cubeCopy.regetSegments(optimizedSegments);
-
+
if (cubeCopy.getSegments().size() != optSegCopy.length * 2) {
throw new IllegalStateException("For cube " + cubeCopy
+ ", every READY segment should be optimized and all segments should be READY before optimizing");
}
-
+
CubeSegment[] originalSegments = new CubeSegment[optSegCopy.length];
int i = 0;
for (CubeSegment seg : optSegCopy) {
@@ -865,7 +871,8 @@ public class CubeManager implements IRealizationProvider {
"For cube " + cubeCopy + ", segment " + seg + " missing StorageLocationIdentifier");
if (StringUtils.isBlank(seg.getLastBuildJobID()))
- throw new IllegalStateException("For cube " + cubeCopy + ", segment " + seg + " missing LastBuildJobID");
+ throw new IllegalStateException(
+ "For cube " + cubeCopy + ", segment " + seg + " missing LastBuildJobID");
seg.setStatus(SegmentStatusEnum.READY);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/16236ef3/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index f7b8275..181e8b9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -370,7 +370,7 @@ public class BuildCubeWithStream {
} catch (Throwable e) {
logger.error("error", e);
exitCode = 1;
- } finally{
+ } finally {
if (buildCubeWithStream != null) {
buildCubeWithStream.after();
buildCubeWithStream.cleanup();
[43/50] [abbrv] kylin git commit: KYLIN-2880 Enhance BadQueryDetector
to include query id
Posted by li...@apache.org.
KYLIN-2880 Enhance BadQueryDetector to include query id
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6a2ee80c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6a2ee80c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6a2ee80c
Branch: refs/heads/sync
Commit: 6a2ee80c9ec19be1d7e72ed8b587fbc48a195867
Parents: c704f7c
Author: Zhong <nj...@apache.org>
Authored: Tue Jan 30 14:13:18 2018 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Jan 30 21:45:18 2018 +0800
----------------------------------------------------------------------
.../kylin/metadata/badquery/BadQueryEntry.java | 14 +++++++++-
.../badquery/BadQueryHistoryManagerTest.java | 15 ++++++----
.../kylin/rest/service/BadQueryDetector.java | 29 +++++++++++---------
.../apache/kylin/rest/service/QueryService.java | 6 ++--
.../rest/service/BadQueryDetectorTest.java | 6 ++--
.../rest/service/KylinQueryTimeoutTest.java | 4 ++-
6 files changed, 49 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
index 1e9311f..56b27ca 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
@@ -47,8 +47,11 @@ public class BadQueryEntry extends RootPersistentEntity implements Comparable<Ba
private String thread;
@JsonProperty("user")
private String user;
+ @JsonProperty("query_id")
+ private String queryId;
- public BadQueryEntry(String sql, String adj, long startTime, float runningSec, String server, String thread, String user) {
+ public BadQueryEntry(String sql, String adj, long startTime, float runningSec, String server, String thread,
+ String user, String queryId) {
this.updateRandomUuid();
this.adj = adj;
this.sql = sql;
@@ -57,12 +60,21 @@ public class BadQueryEntry extends RootPersistentEntity implements Comparable<Ba
this.server = server;
this.thread = thread;
this.user = user;
+ this.queryId = queryId;
}
public BadQueryEntry() {
}
+ public String getQueryId() {
+ return queryId;
+ }
+
+ public void setQueryId(String queryId) {
+ this.queryId = queryId;
+ }
+
public String getUser() {
return user;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
index c4a8229..5001cd2 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.NavigableSet;
+import java.util.UUID;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
@@ -65,7 +66,8 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase {
public void testAddEntryToProject() throws IOException {
KylinConfig kylinConfig = getTestConfig();
BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig);
- BadQueryEntry entry = new BadQueryEntry("sql", "adj", 1459362239992L, 100, "server", "t-0", "user");
+ BadQueryEntry entry = new BadQueryEntry("sql", "adj", 1459362239992L, 100, "server", "t-0", "user",
+ UUID.randomUUID().toString());
BadQueryHistory history = manager.upsertEntryToProject(entry, "default");
NavigableSet<BadQueryEntry> entries = history.getEntries();
assertEquals(4, entries.size());
@@ -81,7 +83,8 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase {
assertEquals("t-0", newEntry.getThread());
for (int i = 0; i < kylinConfig.getBadQueryHistoryNum(); i++) {
- BadQueryEntry tmp = new BadQueryEntry("sql", "adj", 1459362239993L + i, 100 + i, "server", "t-0", "user");
+ BadQueryEntry tmp = new BadQueryEntry("sql", "adj", 1459362239993L + i, 100 + i, "server", "t-0", "user",
+ UUID.randomUUID().toString());
history = manager.upsertEntryToProject(tmp, "default");
}
assertEquals(kylinConfig.getBadQueryHistoryNum(), history.getEntries().size());
@@ -92,14 +95,16 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase {
KylinConfig kylinConfig = getTestConfig();
BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig);
- manager.upsertEntryToProject(new BadQueryEntry("sql", "adj", 1459362239000L, 100, "server", "t-0", "user"),
+ String queryId = UUID.randomUUID().toString();
+ manager.upsertEntryToProject(
+ new BadQueryEntry("sql", "adj", 1459362239000L, 100, "server", "t-0", "user", queryId),
"default");
BadQueryHistory history = manager.upsertEntryToProject(
- new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user"), "default");
+ new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user", queryId), "default");
NavigableSet<BadQueryEntry> entries = history.getEntries();
BadQueryEntry newEntry = entries
- .floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user"));
+ .floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user", queryId));
System.out.println(newEntry);
assertEquals("adj2", newEntry.getAdj());
assertEquals("server2", newEntry.getServer());
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
index 4f7bccf..51f49a7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
@@ -102,15 +102,15 @@ public class BadQueryDetector extends Thread {
for (Notifier notifier : notifiers) {
try {
notifier.badQueryFound(adj, runningSec, //
- e.startTime, e.sqlRequest.getProject(), e.sqlRequest.getSql(), e.user, e.thread);
+ e.startTime, e.sqlRequest.getProject(), e.sqlRequest.getSql(), e.user, e.thread, e.queryId);
} catch (Exception ex) {
logger.error("", ex);
}
}
}
- public void queryStart(Thread thread, SQLRequest sqlRequest, String user) {
- runningQueries.put(thread, new Entry(sqlRequest, user, thread));
+ public void queryStart(Thread thread, SQLRequest sqlRequest, String user, String queryId) {
+ runningQueries.put(thread, new Entry(sqlRequest, user, thread, queryId));
}
public void queryEnd(Thread thread) {
@@ -155,7 +155,7 @@ public class BadQueryDetector extends Thread {
if (runningSec >= alertRunningSec) {
notify(BadQueryEntry.ADJ_SLOW, e);
- dumpStackTrace(e.thread);
+ dumpStackTrace(e.thread, e.queryId);
} else {
break; // entries are sorted by startTime
}
@@ -175,12 +175,13 @@ public class BadQueryDetector extends Thread {
}
// log the stack trace of bad query thread for further analysis
- private void dumpStackTrace(Thread t) {
+ private void dumpStackTrace(Thread t, String queryId) {
int maxStackTraceDepth = kylinConfig.getBadQueryStackTraceDepth();
int current = 0;
StackTraceElement[] stackTrace = t.getStackTrace();
- StringBuilder buf = new StringBuilder("Problematic thread 0x" + Long.toHexString(t.getId()));
+ StringBuilder buf = new StringBuilder(
+ "Problematic thread 0x" + Long.toHexString(t.getId()) + " " + t.getName() + ", query id: " + queryId);
buf.append("\n");
for (StackTraceElement e : stackTrace) {
if (++current > maxStackTraceDepth) {
@@ -193,15 +194,15 @@ public class BadQueryDetector extends Thread {
public interface Notifier {
void badQueryFound(String adj, float runningSec, long startTime, String project, String sql, String user,
- Thread t);
+ Thread t, String queryId);
}
private class LoggerNotifier implements Notifier {
@Override
public void badQueryFound(String adj, float runningSec, long startTime, String project, String sql, String user,
- Thread t) {
- logger.info("{} query has been running {} seconds (project:{}, thread: 0x{}, user:{}) -- {}", adj,
- runningSec, project, Long.toHexString(t.getId()), user, sql);
+ Thread t, String queryId) {
+ logger.info("{} query has been running {} seconds (project:{}, thread: 0x{}, user:{}, query id:{}) -- {}",
+ adj, runningSec, project, Long.toHexString(t.getId()), user, queryId, sql);
}
}
@@ -220,10 +221,10 @@ public class BadQueryDetector extends Thread {
@Override
public void badQueryFound(String adj, float runningSec, long startTime, String project, String sql, String user,
- Thread t) {
+ Thread t, String queryId) {
try {
BadQueryEntry entry = new BadQueryEntry(sql, adj, startTime, runningSec, serverHostname, t.getName(),
- user);
+ user, queryId);
badQueryManager.upsertEntryToProject(entry, project);
} catch (IOException e) {
logger.error("Error in bad query persistence.", e);
@@ -236,12 +237,14 @@ public class BadQueryDetector extends Thread {
final long startTime;
final Thread thread;
final String user;
+ final String queryId;
- Entry(SQLRequest sqlRequest, String user, Thread thread) {
+ Entry(SQLRequest sqlRequest, String user, Thread thread, String queryId) {
this.sqlRequest = sqlRequest;
this.startTime = System.currentTimeMillis();
this.thread = thread;
this.user = user;
+ this.queryId = queryId;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 56fab34..841cad9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -182,11 +182,11 @@ public class QueryService extends BasicService {
return getMetadata(getCubeManager(), project);
}
- public SQLResponse query(SQLRequest sqlRequest) throws Exception {
+ public SQLResponse query(SQLRequest sqlRequest, String queryId) throws Exception {
SQLResponse ret = null;
try {
final String user = SecurityContextHolder.getContext().getAuthentication().getName();
- badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, user);
+ badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, user, queryId);
ret = queryWithSqlMassage(sqlRequest);
return ret;
@@ -492,7 +492,7 @@ public class QueryService extends BasicService {
try {
final boolean isSelect = QueryUtil.isSelectStatement(sqlRequest.getSql());
if (isSelect) {
- sqlResponse = query(sqlRequest);
+ sqlResponse = query(sqlRequest, queryContext.getQueryId());
Trace.addTimelineAnnotation("query almost done");
} else if (kylinConfig.isPushDownEnabled() && kylinConfig.isPushDownUpdateEnabled()) {
sqlResponse = update(sqlRequest);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
index d61dfbe..61e52b7 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
+import java.util.UUID;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.badquery.BadQueryEntry;
@@ -53,7 +54,8 @@ public class BadQueryDetectorTest extends LocalFileMetadataTestCase {
BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000, alertMB, alertRunningSec, 1000);
badQueryDetector.registerNotifier(new BadQueryDetector.Notifier() {
@Override
- public void badQueryFound(String adj, float runningSec, long startTime, String project, String sql, String user, Thread t) {
+ public void badQueryFound(String adj, float runningSec, long startTime, String project, String sql,
+ String user, Thread t, String queryId) {
alerts.add(new String[] { adj, sql });
}
});
@@ -64,7 +66,7 @@ public class BadQueryDetectorTest extends LocalFileMetadataTestCase {
SQLRequest sqlRequest = new SQLRequest();
sqlRequest.setSql(mockSql);
- badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, "user");
+ badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, "user", UUID.randomUUID().toString());
// make sure bad query check happens twice
Thread.sleep((alertRunningSec * 2 + 1) * 1000);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java b/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
index 6a61b2f..38b3270 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
@@ -17,7 +17,9 @@
*/
package org.apache.kylin.rest.service;
+
import java.sql.SQLException;
+import java.util.UUID;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exceptions.KylinTimeoutException;
@@ -69,7 +71,7 @@ public class KylinQueryTimeoutTest extends LocalFileMetadataTestCase {
SQLRequest request = new SQLRequest();
request.setProject("default");
request.setSql("select count(*) from STREAMING_TABLE");
- detector.queryStart(Thread.currentThread(), request, "ADMIN");
+ detector.queryStart(Thread.currentThread(), request, "ADMIN", UUID.randomUUID().toString());
try {
QueryACLTestUtil.mockQuery("default", "select * from STREAMING_TABLE");
} finally{
[48/50] [abbrv] kylin git commit: KYLIN-3194 Fix BrokenExecutable
parsing issue
Posted by li...@apache.org.
KYLIN-3194 Fix BrokenExecutable parsing issue
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/da14ee94
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/da14ee94
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/da14ee94
Branch: refs/heads/sync
Commit: da14ee94bcc5b91fabf99954b88819cc513effea
Parents: f797c84
Author: nichunen <ch...@kyligence.io>
Authored: Sun Jan 28 15:12:17 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Feb 4 08:44:45 2018 +0800
----------------------------------------------------------------------
.../kylin/job/execution/ExecutableManager.java | 26 +++++++++++---------
1 file changed, 15 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/da14ee94/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 0069c03..abcb048 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -501,20 +501,24 @@ public class ExecutableManager {
result.setId(executablePO.getUuid());
result.setName(executablePO.getName());
result.setParams(executablePO.getParams());
- List<ExecutablePO> tasks = executablePO.getTasks();
- if (tasks != null && !tasks.isEmpty()) {
- Preconditions.checkArgument(result instanceof ChainedExecutable);
- for (ExecutablePO subTask : tasks) {
- ((ChainedExecutable) result).addTask(parseTo(subTask));
+
+ if (!(result instanceof BrokenExecutable)) {
+ List<ExecutablePO> tasks = executablePO.getTasks();
+ if (tasks != null && !tasks.isEmpty()) {
+ Preconditions.checkArgument(result instanceof ChainedExecutable);
+ for (ExecutablePO subTask : tasks) {
+ ((ChainedExecutable) result).addTask(parseTo(subTask));
+ }
}
- }
- List<ExecutablePO> tasksForCheck = executablePO.getTasksForCheck();
- if (tasksForCheck != null && !tasksForCheck.isEmpty()) {
- Preconditions.checkArgument(result instanceof CheckpointExecutable);
- for (ExecutablePO subTaskForCheck : tasksForCheck) {
- ((CheckpointExecutable) result).addTaskForCheck(parseTo(subTaskForCheck));
+ List<ExecutablePO> tasksForCheck = executablePO.getTasksForCheck();
+ if (tasksForCheck != null && !tasksForCheck.isEmpty()) {
+ Preconditions.checkArgument(result instanceof CheckpointExecutable);
+ for (ExecutablePO subTaskForCheck : tasksForCheck) {
+ ((CheckpointExecutable) result).addTaskForCheck(parseTo(subTaskForCheck));
+ }
}
}
+
return result;
}
[08/50] [abbrv] kylin git commit: Update exec-maven-plugin version to
1.6.0
Posted by li...@apache.org.
Update exec-maven-plugin version to 1.6.0
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cc130e92
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cc130e92
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cc130e92
Branch: refs/heads/sync
Commit: cc130e925622bd962674b4125d9b93f62ebb12bf
Parents: 8e8d71b
Author: nichunen <ch...@kyligence.io>
Authored: Thu Jan 4 21:32:15 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
external/pom.xml | 2 +-
pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cc130e92/external/pom.xml
----------------------------------------------------------------------
diff --git a/external/pom.xml b/external/pom.xml
index 2cc09b2..8e0a40a 100644
--- a/external/pom.xml
+++ b/external/pom.xml
@@ -47,7 +47,7 @@
<plugin>
<artifactId>exec-maven-plugin</artifactId>
<groupId>org.codehaus.mojo</groupId>
- <version>1.5.0</version>
+ <version>1.6.0</version>
<executions>
<execution>
<id>install_htrace</id>
http://git-wip-us.apache.org/repos/asf/kylin/blob/cc130e92/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index da43c62..be2d2bf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1096,7 +1096,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.5.0</version>
+ <version>1.6.0</version>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
[39/50] [abbrv] kylin git commit: KYLIN-3210 The project shows
'_null' in result page.
Posted by li...@apache.org.
KYLIN-3210 The project shows '_null' in result page.
Signed-off-by: chenzhx <ch...@apache.org>
KYLIN-3213 Kylin help has duplicate items
Signed-off-by: chenzhx <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4c31b1d4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4c31b1d4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4c31b1d4
Branch: refs/heads/sync
Commit: 4c31b1d4a582522a7a2109bed470d4f65a145c44
Parents: 0f63fa6
Author: xingpeng <xi...@zte.com.cn>
Authored: Mon Jan 29 18:00:14 2018 +0800
Committer: chenzhx <ch...@apache.org>
Committed: Tue Jan 30 16:59:42 2018 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/query.js | 2 +-
webapp/app/js/services/kylinProperties.js | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c31b1d4/webapp/app/js/controllers/query.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/query.js b/webapp/app/js/controllers/query.js
index e9ea614..0637000 100644
--- a/webapp/app/js/controllers/query.js
+++ b/webapp/app/js/controllers/query.js
@@ -61,7 +61,7 @@ KylinApp
var query = {
originSql: sql,
sql: sql,
- project: (!!project)? project:$scope.projectModel.selectedProject,
+ project: (!!project)? project:$scope.projectModel.getSelectedProject(),
status: 'executing',
acceptPartial: true,
result: {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4c31b1d4/webapp/app/js/services/kylinProperties.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/kylinProperties.js b/webapp/app/js/services/kylinProperties.js
index ec23397..c344866 100644
--- a/webapp/app/js/services/kylinProperties.js
+++ b/webapp/app/js/services/kylinProperties.js
@@ -94,6 +94,7 @@ KylinApp.service('kylinConfig', function (AdminService, $log) {
Config.reference_links.hadoop.link = this.getProperty("kylin.web.link-hadoop").trim();
Config.reference_links.diagnostic.link = this.getProperty("kylin.web.link-diagnostic").trim();
Config.contact_mail = this.getProperty("kylin.web.contact-mail").trim();
+ Config.documents = [];
var doc_length = this.getProperty("kylin.web.help.length").trim();
for (var i = 0; i < doc_length; i++) {
var _doc = {};
[50/50] [abbrv] kylin git commit: KYLIN-3211 add Exception message
for the illegal IntDimEnc length
Posted by li...@apache.org.
KYLIN-3211 add Exception message for the illegal IntDimEnc length
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/04b32207
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/04b32207
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/04b32207
Branch: refs/heads/sync
Commit: 04b3220739713a7d157e6a286fabea5a5036fad4
Parents: 4ade239
Author: Yifei Wu <va...@gmail.com>
Authored: Mon Jan 29 21:20:08 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Feb 4 08:45:40 2018 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/dimension/IntDimEnc.java | 10 ++++++----
.../java/org/apache/kylin/dimension/IntDimEncTest.java | 5 +++--
2 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/04b32207/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
index 3650200..6ce003b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
@@ -34,12 +34,13 @@ import org.slf4j.LoggerFactory;
* deprecated use IntegerDimEnc instead
* @deprecated
*/
-public class IntDimEnc extends DimensionEncoding implements Serializable{
+public class IntDimEnc extends DimensionEncoding implements Serializable {
private static final long serialVersionUID = 1L;
private static Logger logger = LoggerFactory.getLogger(IntDimEnc.class);
- private static final long[] CAP = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL, 0xffffffffffffffL, Long.MAX_VALUE };
+ private static final long[] CAP = { 0, 0xffL, 0xffffL, 0xffffffL, 0xffffffffL, 0xffffffffffL, 0xffffffffffffL,
+ 0xffffffffffffffL, Long.MAX_VALUE };
public static final String ENCODING_NAME = "int";
@@ -67,7 +68,7 @@ public class IntDimEnc extends DimensionEncoding implements Serializable{
public IntDimEnc(int len) {
if (len <= 0 || len >= CAP.length)
- throw new IllegalArgumentException();
+ throw new IllegalArgumentException("the length of IntDimEnc is " + len + ", which should be 1-8");
this.fixedLen = len;
}
@@ -87,7 +88,8 @@ public class IntDimEnc extends DimensionEncoding implements Serializable{
long integer = Long.parseLong(valueStr);
if (integer > CAP[fixedLen]) {
if (avoidVerbose++ % 10000 == 0) {
- logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:" + avoidVerbose);
+ logger.warn("Expect at most " + fixedLen + " bytes, but got " + valueStr + ", will truncate, hit times:"
+ + avoidVerbose);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/04b32207/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java b/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java
index 0f55b3e..fc4163b 100644
--- a/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/dimension/IntDimEncTest.java
@@ -31,18 +31,19 @@ import org.junit.Test;
public class IntDimEncTest {
@Test
- public void testConstructor() {
+ public void testBadConstructor() {
try {
new IntDimEnc(0);
Assert.fail();
} catch (IllegalArgumentException e) {
// expect
+ Assert.assertEquals("the length of IntDimEnc is 0, which should be 1-8", e.getMessage());
}
try {
new IntDimEnc(9);
Assert.fail();
} catch (IllegalArgumentException e) {
- // expect
+ Assert.assertEquals("the length of IntDimEnc is 9, which should be 1-8", e.getMessage());
}
new IntDimEnc(8);
}
[42/50] [abbrv] kylin git commit: KYLIN-2908 Add one option for
migration tool to indicate whether to migrate segment data
Posted by li...@apache.org.
KYLIN-2908 Add one option for migration tool to indicate whether to migrate segment data
Signed-off-by: Zhong <nj...@apache.org>
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c704f7cd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c704f7cd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c704f7cd
Branch: refs/heads/sync
Commit: c704f7cda79921c5a22645492a52c94734847541
Parents: fee5730
Author: Wang Vic <jw...@ebay.com>
Authored: Wed Sep 27 11:18:22 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Jan 30 19:10:51 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 6 +
.../org/apache/kylin/tool/CubeMigrationCLI.java | 114 +++++++++++++------
2 files changed, 87 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c704f7cd/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 27918bc..ea5006e 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -383,6 +383,12 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
this.createTimeUTC = createTimeUTC;
}
+ public void clearCuboids() {
+ cuboidBytes = null;
+ cuboidBytesRecommend = null;
+ cuboidLastOptimized = 0L;
+ }
+
public Set<Long> getCuboidsByMode(String cuboidModeName) {
return getCuboidsByMode(cuboidModeName == null ? null : CuboidModeEnum.getByModeName(cuboidModeName));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c704f7cd/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index f95139e..a4a6ab9 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -90,6 +90,7 @@ public class CubeMigrationCLI extends AbstractApplication {
private HBaseAdmin hbaseAdmin;
protected boolean doAclCopy = false;
protected boolean doOverwrite = false;
+ protected boolean doMigrateSegment = true;
protected String dstProject;
private static final String ACL_PREFIX = "/acl/";
@@ -97,16 +98,20 @@ public class CubeMigrationCLI extends AbstractApplication {
public static void main(String[] args) throws IOException, InterruptedException {
CubeMigrationCLI cli = new CubeMigrationCLI();
- if (args.length != 8) {
+ if (args.length != 8 && args.length != 9) {
cli.usage();
System.exit(1);
}
- cli.moveCube(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
+ if (args.length == 8) {
+ cli.moveCube(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
+ } else if (args.length == 9) {
+ cli.moveCube(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7], args[8]);
+ }
}
protected void usage() {
System.out.println(
- "Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName copyAclOrNot purgeOrNot overwriteIfExists realExecute");
+ "Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName copyAclOrNot purgeOrNot overwriteIfExists realExecute migrateSegmentOrNot");
System.out.println("srcKylinConfigUri: The KylinConfig of the cube’s source \n"
+ "dstKylinConfigUri: The KylinConfig of the cube’s new home \n"
+ "cubeName: the name of cube to be migrated. \n"
@@ -114,16 +119,36 @@ public class CubeMigrationCLI extends AbstractApplication {
+ "copyAclOrNot: true or false: whether copy cube ACL to target environment. \n"
+ "purgeOrNot: true or false: whether purge the cube from src server after the migration. \n"
+ "overwriteIfExists: overwrite cube if it already exists in the target environment. \n"
- + "realExecute: if false, just print the operations to take, if true, do the real migration. \n");
+ + "realExecute: if false, just print the operations to take, if true, do the real migration. \n"
+ + "migrateSegmentOrNot:(optional) true or false: whether copy segment data to target environment. \n");
}
- public void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String copyAcl,
+ public void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl,
String purgeAndDisable, String overwriteIfExists, String realExecute)
throws IOException, InterruptedException {
- doAclCopy = Boolean.parseBoolean(copyAcl);
- doOverwrite = Boolean.parseBoolean(overwriteIfExists);
+ moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName,
+ projectName, Boolean.parseBoolean(copyAcl), Boolean.parseBoolean(purgeAndDisable),
+ Boolean.parseBoolean(overwriteIfExists), Boolean.parseBoolean(realExecute), true);
+ }
+
+ public void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl,
+ String purgeAndDisable, String overwriteIfExists, String realExecute, String migrateSegment)
+ throws IOException, InterruptedException {
+
+ moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName,
+ projectName, Boolean.parseBoolean(copyAcl), Boolean.parseBoolean(purgeAndDisable),
+ Boolean.parseBoolean(overwriteIfExists), Boolean.parseBoolean(realExecute),
+ Boolean.parseBoolean(migrateSegment));
+ }
+
+ public void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, boolean copyAcl,
+ boolean purgeAndDisable, boolean overwriteIfExists, boolean realExecute, boolean migrateSegment)
+ throws IOException, InterruptedException {
+ doAclCopy = copyAcl;
+ doOverwrite = overwriteIfExists;
+ doMigrateSegment = migrateSegment;
srcConfig = srcCfg;
srcStore = ResourceStore.getStore(srcConfig);
dstConfig = dstCfg;
@@ -134,7 +159,10 @@ public class CubeMigrationCLI extends AbstractApplication {
CubeInstance cube = cubeManager.getCube(cubeName);
logger.info("cube to be moved is : " + cubeName);
- checkCubeState(cube);
+ if (migrateSegment) {
+ checkCubeState(cube);
+ }
+
checkAndGetHbaseUrl();
Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
@@ -142,31 +170,29 @@ public class CubeMigrationCLI extends AbstractApplication {
hdfsFS = HadoopUtil.getWorkingFileSystem();
operations = new ArrayList<Opt>();
copyFilesInMetaStore(cube);
- renameFoldersInHdfs(cube);
- changeHtableHost(cube);
+ if (migrateSegment) {
+ renameFoldersInHdfs(cube);
+ changeHtableHost(cube);
+ } else {
+ clearSegments(cubeName); // this should be after copyFilesInMetaStore
+ }
addCubeAndModelIntoProject(cube, cubeName);
- if (Boolean.parseBoolean(purgeAndDisable) == true) {
+ if (migrateSegment && purgeAndDisable) {
purgeAndDisable(cubeName); // this should be the last action
}
- if (Boolean.parseBoolean(realExecute) == true) {
+ if (realExecute) {
doOpts();
- checkMigrationSuccess(dstConfig, cubeName, true);
+ if (migrateSegment) {
+ checkMigrationSuccess(dstConfig, cubeName, true);
+ }
updateMeta(dstConfig);
} else {
showOpts();
}
}
- public void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl,
- String purgeAndDisable, String overwriteIfExists, String realExecute)
- throws IOException, InterruptedException {
-
- moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName,
- projectName, copyAcl, purgeAndDisable, overwriteIfExists, realExecute);
- }
-
public void checkMigrationSuccess(KylinConfig kylinConfig, String cubeName, Boolean ifFix) throws IOException {
CubeMigrationCheckCLI checkCLI = new CubeMigrationCheckCLI(kylinConfig, ifFix);
checkCLI.execute(cubeName);
@@ -215,22 +241,28 @@ public class CubeMigrationCLI extends AbstractApplication {
}
}
- protected void copyFilesInMetaStore(CubeInstance cube) throws IOException {
+ protected void clearSegments(String cubeName) throws IOException {
+ operations.add(new Opt(OptType.CLEAR_SEGMENTS, new Object[] { cubeName }));
+ }
- List<String> metaItems = new ArrayList<String>();
- Set<String> dictAndSnapshot = new HashSet<String>();
- listCubeRelatedResources(cube, metaItems, dictAndSnapshot);
+ protected void copyFilesInMetaStore(CubeInstance cube) throws IOException {
if (dstStore.exists(cube.getResourcePath()) && !doOverwrite)
throw new IllegalStateException("The cube named " + cube.getName()
+ " already exists on target metadata store. Use overwriteIfExists to overwrite it");
+ List<String> metaItems = new ArrayList<String>();
+ Set<String> dictAndSnapshot = new HashSet<String>();
+ listCubeRelatedResources(cube, metaItems, dictAndSnapshot);
+
for (String item : metaItems) {
operations.add(new Opt(OptType.COPY_FILE_IN_META, new Object[] { item }));
}
- for (String item : dictAndSnapshot) {
- operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
+ if (doMigrateSegment) {
+ for (String item : dictAndSnapshot) {
+ operations.add(new Opt(OptType.COPY_DICT_OR_SNAPSHOT, new Object[] { item, cube.getName() }));
+ }
}
}
@@ -285,10 +317,12 @@ public class CubeMigrationCLI extends AbstractApplication {
metaResource.addAll(getCompatibleTablePath(tblRefs, prj, ResourceStore.TABLE_RESOURCE_ROOT));
metaResource.addAll(getCompatibleTablePath(tblRefs, prj, ResourceStore.TABLE_EXD_RESOURCE_ROOT));
- for (CubeSegment segment : cube.getSegments()) {
- metaResource.add(segment.getStatisticsResourcePath());
- dictAndSnapshot.addAll(segment.getSnapshotPaths());
- dictAndSnapshot.addAll(segment.getDictionaryPaths());
+ if (doMigrateSegment) {
+ for (CubeSegment segment : cube.getSegments()) {
+ metaResource.add(segment.getStatisticsResourcePath());
+ dictAndSnapshot.addAll(segment.getSnapshotPaths());
+ dictAndSnapshot.addAll(segment.getDictionaryPaths());
+ }
}
if (doAclCopy) {
@@ -308,7 +342,7 @@ public class CubeMigrationCLI extends AbstractApplication {
}
protected enum OptType {
- COPY_FILE_IN_META, COPY_DICT_OR_SNAPSHOT, RENAME_FOLDER_IN_HDFS, ADD_INTO_PROJECT, CHANGE_HTABLE_HOST, PURGE_AND_DISABLE
+ COPY_FILE_IN_META, COPY_DICT_OR_SNAPSHOT, RENAME_FOLDER_IN_HDFS, ADD_INTO_PROJECT, CHANGE_HTABLE_HOST, PURGE_AND_DISABLE, CLEAR_SEGMENTS
}
protected void addOpt(OptType type, Object[] params) {
@@ -490,7 +524,7 @@ public class CubeMigrationCLI extends AbstractApplication {
project.addTable(tableRef.getTableIdentity());
}
- if (project.getModels().contains(modelName) == false)
+ if (!project.getModels().contains(modelName))
project.addModel(modelName);
project.removeRealization(RealizationType.CUBE, cubeName);
project.addRealizationEntry(RealizationType.CUBE, cubeName);
@@ -499,6 +533,20 @@ public class CubeMigrationCLI extends AbstractApplication {
logger.info("Project instance for " + projectName + " is corrected");
break;
}
+ case CLEAR_SEGMENTS: {
+ String cubeName = (String) opt.params[0];
+ String cubeInstancePath = CubeInstance.concatResourcePath(cubeName);
+ Serializer<CubeInstance> cubeInstanceSerializer = new JsonSerializer<CubeInstance>(CubeInstance.class);
+ CubeInstance cubeInstance = dstStore.getResource(cubeInstancePath, CubeInstance.class,
+ cubeInstanceSerializer);
+ cubeInstance.getSegments().clear();
+ cubeInstance.clearCuboids();
+ cubeInstance.setCreateTimeUTC(System.currentTimeMillis());
+ cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
+ dstStore.putResource(cubeInstancePath, cubeInstance, cubeInstanceSerializer);
+ logger.info("Cleared segments for " + cubeName + ", since segments has not been copied");
+ break;
+ }
case PURGE_AND_DISABLE: {
String cubeName = (String) opt.params[0];
String cubeResPath = CubeInstance.concatResourcePath(cubeName);
[34/50] [abbrv] kylin git commit: KYLIN-2909 refactor to host
templates in each modules
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-job/src/main/resources/mail_templates/JOB_ERROR.ftl
----------------------------------------------------------------------
diff --git a/core-job/src/main/resources/mail_templates/JOB_ERROR.ftl b/core-job/src/main/resources/mail_templates/JOB_ERROR.ftl
new file mode 100644
index 0000000..6012037
--- /dev/null
+++ b/core-job/src/main/resources/mail_templates/JOB_ERROR.ftl
@@ -0,0 +1,372 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+<span style="
+ line-height: 1.1;font-size: 18px;">
+ <p style="text-align:left;">Dear Kylin user,</p>
+ <p>This cube <strong>failure</strong> may be caused by backend platform being busy, please try again.</p>
+ <p>Thank you for using Kylin and we apologize for the inconvenience.</p>
+</span>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h1>
+ <span style="display: inline;
+ background-color: #d9534f;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Error </span>
+ </h1>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+
+ <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #ebccd1;">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${job_name}
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${env_name}
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Submitter
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${submitter}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Job Engine
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${job_engine}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Project
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${project_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Cube Name
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${cube_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Source Records Count
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${source_records_count}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Start Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${start_time}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Duration
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${duration}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">MR Waiting Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${mr_waiting}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Last Update Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${last_update_time}
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Job Error Details
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Error Step
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${error_step}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ MR Job
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${mr_job_id}
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Logs
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+
+ <pre style="white-space: pre-wrap;">${error_log}</pre>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ </table>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h4 style="font-weight: 500;
+ line-height: 1.1;font-size:18px;">
+ <p>Best Wishes!</p>
+ <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-job/src/main/resources/mail_templates/JOB_SUCCEED.ftl
----------------------------------------------------------------------
diff --git a/core-job/src/main/resources/mail_templates/JOB_SUCCEED.ftl b/core-job/src/main/resources/mail_templates/JOB_SUCCEED.ftl
new file mode 100644
index 0000000..f1fb45c
--- /dev/null
+++ b/core-job/src/main/resources/mail_templates/JOB_SUCCEED.ftl
@@ -0,0 +1,274 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+<span style="line-height: 1.1;font-size: 18px;">
+ <p style="text-align:left;">Dear Kylin user,</p>
+ <p>Congratulations! Please feel free to query based on kylin cube. Thank you for using Kylin.</p>
+</span>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h1>
+ <span style="display: inline;
+ background-color: #5cb85c;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Succeed </span>
+ </h1>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #d6e9c6;">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #dff0d8;
+ border:1px solid #d6e9c6;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #3c763d;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${job_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #dff0d8;
+ border:1px solid #d6e9c6;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #3c763d;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${env_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Submitter
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${submitter}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Job Engine
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${job_engine}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Project
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${project_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Cube Name
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${cube_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Source Records Count
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${source_records_count}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Start Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${start_time}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Duration
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${duration}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">MR Waiting Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${mr_waiting}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Last Update Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${last_update_time}
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ </table>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h4 style="font-weight: 500;
+ line-height: 1.1;font-size:18px;">
+ <p>Best Wishes!</p>
+ <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-job/src/main/resources/mail_templates/METADATA_PERSIST_FAIL.ftl
----------------------------------------------------------------------
diff --git a/core-job/src/main/resources/mail_templates/METADATA_PERSIST_FAIL.ftl b/core-job/src/main/resources/mail_templates/METADATA_PERSIST_FAIL.ftl
new file mode 100644
index 0000000..2511b57
--- /dev/null
+++ b/core-job/src/main/resources/mail_templates/METADATA_PERSIST_FAIL.ftl
@@ -0,0 +1,179 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <span style="line-height: 1.1;font-size: 18px;">
+ <p style="text-align:left;">Dear Kylin user,</p>
+ <p>Kylin fails to update the job output due to some hbase issue. Need to ask Hadoop Service Team for help as soon as possible.</p>
+</span>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h1>
+ <span style="display: inline;
+ background-color: #d9534f;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Error </span>
+ </h1>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #ebccd1;">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${job_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${env_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Submitter
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${submitter}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Job Engine
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${job_engine}
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Logs
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ <pre style="white-space: pre-wrap;">${error_log}</pre>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ </table>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h4 style="font-weight: 500;
+ line-height: 1.1;font-size:18px;">
+ <p>Best Wishes!</p>
+ <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-job/src/test/java/org/apache/kylin/job/util/MailNotificationUtilTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/util/MailNotificationUtilTest.java b/core-job/src/test/java/org/apache/kylin/job/util/MailNotificationUtilTest.java
new file mode 100644
index 0000000..50627e0
--- /dev/null
+++ b/core-job/src/test/java/org/apache/kylin/job/util/MailNotificationUtilTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.util;
+
+import org.apache.kylin.job.execution.ExecutableState;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+public class MailNotificationUtilTest {
+
+ @Test
+ public void testGetMailTitle() {
+ String[] titleParts = new String[] { "JOB", "SUCCEED" };
+ Assert.assertEquals("[" + titleParts[0] + "]-[" + titleParts[1] + "]",
+ MailNotificationUtil.getMailTitle(titleParts));
+ }
+
+ @Test
+ public void testHasMailNotification() {
+ Assert.assertTrue(MailNotificationUtil.hasMailNotification(ExecutableState.DISCARDED));
+ Assert.assertTrue(MailNotificationUtil.hasMailNotification(ExecutableState.ERROR));
+ Assert.assertTrue(MailNotificationUtil.hasMailNotification(ExecutableState.SUCCEED));
+ Assert.assertFalse(MailNotificationUtil.hasMailNotification(ExecutableState.RUNNING));
+ Assert.assertFalse(MailNotificationUtil.hasMailNotification(ExecutableState.STOPPED));
+ Assert.assertFalse(MailNotificationUtil.hasMailNotification(ExecutableState.READY));
+ }
+
+ @Test
+ public void testGetMailContent() {
+ Assert.assertFalse(
+ MailNotificationUtil.getMailContent(ExecutableState.DISCARDED, Maps.<String, Object>newHashMap())
+ .startsWith("Cannot find email template for"));
+ Assert.assertFalse(MailNotificationUtil.getMailContent(ExecutableState.ERROR, Maps.<String, Object>newHashMap())
+ .startsWith("Cannot find email template for"));
+ Assert.assertFalse(
+ MailNotificationUtil.getMailContent(ExecutableState.SUCCEED, Maps.<String, Object>newHashMap())
+ .startsWith("Cannot find email template for"));
+ Assert.assertTrue(
+ MailNotificationUtil.getMailContent(ExecutableState.RUNNING, Maps.<String, Object>newHashMap())
+ .startsWith("Cannot find email template for"));
+ Assert.assertTrue(
+ MailNotificationUtil.getMailContent(ExecutableState.STOPPED, Maps.<String, Object>newHashMap())
+ .startsWith("Cannot find email template for"));
+ Assert.assertTrue(MailNotificationUtil.getMailContent(ExecutableState.READY, Maps.<String, Object>newHashMap())
+ .startsWith("Cannot find email template for"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 548dcb9..9173dbd 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -27,10 +27,7 @@ import java.util.Map;
import java.util.TimeZone;
import java.util.regex.Matcher;
-
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.EmailTemplateEnum;
-import org.apache.kylin.common.util.EmailTemplateFactory;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
@@ -47,7 +44,7 @@ import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.job.execution.Output;
import org.apache.kylin.job.metrics.JobMetricsFacade;
-import org.apache.kylin.job.util.ExecutableStateUtil;
+import org.apache.kylin.job.util.MailNotificationUtil;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
@@ -205,8 +202,7 @@ public class CubingJob extends DefaultChainedExecutable {
return null;
}
- EmailTemplateEnum templateEnum = ExecutableStateUtil.getEmailTemplateEnum(state);
- if (templateEnum == null) {
+ if (!MailNotificationUtil.hasMailNotification(state)) {
logger.info("Cannot find email template for job state: " + state);
return null;
}
@@ -215,7 +211,7 @@ public class CubingJob extends DefaultChainedExecutable {
dataMap.put("job_name", getName());
dataMap.put("env_name", getDeployEnvName());
dataMap.put("submitter", StringUtil.noBlank(getSubmitter(), "missing submitter"));
- dataMap.put("job_engine", EmailTemplateFactory.getLocalHostName());
+ dataMap.put("job_engine", MailNotificationUtil.getLocalHostName());
dataMap.put("project_name", getProjectName());
dataMap.put("cube_name", cubeInstance.getName());
dataMap.put("source_records_count", String.valueOf(findSourceRecordCount()));
@@ -242,15 +238,14 @@ public class CubingJob extends DefaultChainedExecutable {
final String mrJobId = errorOutput.getExtra().get(ExecutableConstants.MR_JOB_ID);
dataMap.put("mr_job_id", StringUtil.noBlank(mrJobId, "Not initialized"));
} else {
- dataMap.put("mr_job_id", EmailTemplateFactory.NA);
+ dataMap.put("mr_job_id", MailNotificationUtil.NA);
}
dataMap.put("error_log",
Matcher.quoteReplacement(StringUtil.noBlank(output.getVerboseMsg(), "no error message")));
}
- String content = EmailTemplateFactory.getInstance()
- .buildEmailContent(ExecutableStateUtil.getEmailTemplateEnum(state), dataMap);
- String title = EmailTemplateFactory.getEmailTitle("JOB", state.toString(), getDeployEnvName(), getProjectName(),
+ String content = MailNotificationUtil.getMailContent(state, dataMap);
+ String title = MailNotificationUtil.getMailTitle("JOB", state.toString(), getDeployEnvName(), getProjectName(),
cubeInstance.getName());
return Pair.newPair(title, content);
}
@@ -279,9 +274,8 @@ public class CubingJob extends DefaultChainedExecutable {
protected void updateMetrics(ExecutableContext context, ExecuteResult result, ExecutableState state) {
JobMetricsFacade.JobStatisticsResult jobStats = new JobMetricsFacade.JobStatisticsResult();
- jobStats.setWrapper(getSubmitter(), getProjectName(),
- CubingExecutableUtil.getCubeName(getParams()), getId(), getJobType(),
- getAlgorithm() == null ? "NULL" : getAlgorithm().toString());
+ jobStats.setWrapper(getSubmitter(), getProjectName(), CubingExecutableUtil.getCubeName(getParams()), getId(),
+ getJobType(), getAlgorithm() == null ? "NULL" : getAlgorithm().toString());
if (state == ExecutableState.SUCCEED) {
jobStats.setJobStats(findSourceSizeBytes(), findCubeSizeBytes(), getDuration(), getMapReduceWaitTime(),
[22/50] [abbrv] kylin git commit: Minor, fix model draft not listed
Posted by li...@apache.org.
Minor, fix model draft not listed
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/67f7197b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/67f7197b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/67f7197b
Branch: refs/heads/sync
Commit: 67f7197b849249df65d79bf3ec3ca090c0bb50a9
Parents: cc130e9
Author: nichunen <ch...@kyligence.io>
Authored: Fri Jan 5 17:05:41 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/service/ModelService.java | 2 +-
.../kylin/rest/service/ModelServiceTest.java | 18 ++++++++++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/67f7197b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 43ce0b1..ee01f3e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -413,7 +413,7 @@ public class ModelService extends BasicService {
RootPersistentEntity e = d.getEntity();
if (e instanceof DataModelDesc) {
DataModelDesc m = (DataModelDesc) e;
- if (modelName == null || modelName.equals(m.getName()))
+ if (StringUtils.isEmpty(modelName) || modelName.equals(m.getName()))
result.add(d);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/67f7197b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index fff4ed9..aeb6d79 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -27,6 +27,8 @@ import java.util.List;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.job.exception.JobException;
+import org.apache.kylin.metadata.draft.Draft;
+import org.apache.kylin.metadata.draft.DraftManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.ModelDimensionDesc;
import org.junit.Assert;
@@ -132,6 +134,22 @@ public class ModelServiceTest extends ServiceTestBase {
}
}
+ @Test
+ public void testModelDraft() throws IOException {
+ DraftManager mgr = DraftManager.getInstance(getTestConfig());
+ // Create a draft of model
+ Draft d = new Draft();
+ d.setProject("default");
+ d.updateRandomUuid();
+ DataModelDesc modelDesc = modelService.getModel("ci_left_join_model", "default");
+ d.setEntity(modelDesc);
+ mgr.save(d);
+
+ // Check list draft
+ List<Draft> draftList = modelService.listModelDrafts("", "default");
+ Assert.assertEquals(draftList.size(), 1);
+ }
+
private String[] cutItems(String[] origin, int count) {
if (origin == null)
[29/50] [abbrv] kylin git commit: KYLIN-3158 Retry failed the
metadata sync event on the failed node only
Posted by li...@apache.org.
KYLIN-3158 Retry failed the metadata sync event on the failed node only
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/baf7133d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/baf7133d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/baf7133d
Branch: refs/heads/sync
Commit: baf7133dabd8f1e53f5235b61da2bc5da334d870
Parents: 0fce61c
Author: Li Yang <li...@apache.org>
Authored: Fri Jan 12 21:46:47 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +
.../kylin/metadata/cachesync/Broadcaster.java | 127 ++++++++++++++-----
.../metadata/cachesync/SingleValueCache.java | 6 +-
.../metadata/cachesync/BroadcasterTest.java | 43 +++++++
.../kylin/rest/init/InitialTaskManager.java | 2 +-
.../apache/kylin/rest/service/CacheService.java | 4 +-
6 files changed, 151 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/baf7133d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b053daa..5045b2f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -303,6 +303,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.metadata.sync-retries", "3"));
}
+ public String getCacheSyncErrorHandler() {
+ return getOptional("kylin.metadata.sync-error-handler");
+ }
+
// for test only
public void setMetadataUrl(String metadataUrl) {
setProperty("kylin.metadata.url", metadataUrl);
http://git-wip-us.apache.org/repos/asf/kylin/blob/baf7133d/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 05910ea..6462a27 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.restclient.RestClient;
+import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.DaemonThreadFactory;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
@@ -68,7 +69,7 @@ public class Broadcaster {
public static Broadcaster getInstance(KylinConfig config) {
return config.getManager(Broadcaster.class);
}
-
+
// called by reflection
static Broadcaster newInstance(KylinConfig config) {
return new Broadcaster(config);
@@ -79,13 +80,19 @@ public class Broadcaster {
static final Map<String, List<Listener>> staticListenerMap = Maps.newConcurrentMap();
private KylinConfig config;
+ private ExecutorService announceMainLoop;
+ private ExecutorService announceThreadPool;
+ private SyncErrorHandler syncErrorHandler;
private BlockingDeque<BroadcastEvent> broadcastEvents = new LinkedBlockingDeque<>();
private Map<String, List<Listener>> listenerMap = Maps.newConcurrentMap();
- private AtomicLong counter = new AtomicLong();
-
+ private AtomicLong counter = new AtomicLong(); // a counter for testing purpose
+
private Broadcaster(final KylinConfig config) {
this.config = config;
- final int retryLimitTimes = config.getCacheSyncRetrys();
+ this.syncErrorHandler = getSyncErrorHandler(config);
+ this.announceMainLoop = Executors.newSingleThreadExecutor(new DaemonThreadFactory());
+ this.announceThreadPool = new ThreadPoolExecutor(1, 10, 60L, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>(), new DaemonThreadFactory());
final String[] nodes = config.getRestServers();
if (nodes == null || nodes.length < 1) {
@@ -93,21 +100,14 @@ public class Broadcaster {
}
logger.debug(nodes.length + " nodes in the cluster: " + Arrays.toString(nodes));
- Executors.newSingleThreadExecutor(new DaemonThreadFactory()).execute(new Runnable() {
+ announceMainLoop.execute(new Runnable() {
@Override
public void run() {
final Map<String, RestClient> restClientMap = Maps.newHashMap();
- final ExecutorService wipingCachePool = new ThreadPoolExecutor(1, 10, 60L, TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(), new DaemonThreadFactory());
- while (true) {
+ while (!announceThreadPool.isShutdown()) {
try {
final BroadcastEvent broadcastEvent = broadcastEvents.takeFirst();
- broadcastEvent.setRetryTime(broadcastEvent.getRetryTime() + 1);
- if (broadcastEvent.getRetryTime() > retryLimitTimes) {
- logger.info("broadcastEvent retry up to limit times, broadcastEvent:{}", broadcastEvent);
- continue;
- }
String[] restServers = config.getRestServers();
logger.debug("Servers in the cluster: " + Arrays.toString(restServers));
@@ -117,25 +117,27 @@ public class Broadcaster {
}
}
- logger.debug("Announcing new broadcast event: " + broadcastEvent);
+ String toWhere = broadcastEvent.getTargetNode();
+ if (toWhere == null)
+ toWhere = "all";
+ logger.debug("Announcing new broadcast to " + toWhere + ": " + broadcastEvent);
+
for (final String node : restServers) {
- wipingCachePool.execute(new Runnable() {
+ if (!(toWhere.equals("all") || toWhere.equals(node)))
+ continue;
+
+ announceThreadPool.execute(new Runnable() {
@Override
public void run() {
+ RestClient restClient = restClientMap.get(node);
try {
- restClientMap.get(node).wipeCache(broadcastEvent.getEntity(),
- broadcastEvent.getEvent(), broadcastEvent.getCacheKey());
+ restClient.wipeCache(broadcastEvent.getEntity(), broadcastEvent.getEvent(),
+ broadcastEvent.getCacheKey());
} catch (IOException e) {
- logger.warn("Thread failed during wipe cache at {}, error msg: {}",
- broadcastEvent, e);
- // when sync failed, put back to queue
- try {
- broadcastEvents.putLast(broadcastEvent);
- } catch (InterruptedException ex) {
- logger.warn(
- "error reentry failed broadcastEvent to queue, broacastEvent:{}, error: {} ",
- broadcastEvent, ex);
- }
+ logger.error(
+ "Announce broadcast event failed, targetNode {} broadcastEvent {}, error msg: {}",
+ node, broadcastEvent, e);
+ syncErrorHandler.handleAnnounceError(node, restClient, broadcastEvent);
}
}
});
@@ -148,6 +150,23 @@ public class Broadcaster {
});
}
+ private SyncErrorHandler getSyncErrorHandler(KylinConfig config) {
+ String clzName = config.getCacheSyncErrorHandler();
+ if (StringUtils.isEmpty(clzName)) {
+ clzName = DefaultSyncErrorHandler.class.getName();
+ }
+ return (SyncErrorHandler) ClassUtil.newInstance(clzName);
+ }
+
+ public KylinConfig getConfig() {
+ return config;
+ }
+
+ public void stopAnnounce() {
+ announceThreadPool.shutdown();
+ announceMainLoop.shutdown();
+ }
+
// static listener survives cache wipe and goes after normal listeners
public void registerStaticListener(Listener listener, String... entities) {
doRegisterListener(staticListenerMap, listener, entities);
@@ -263,15 +282,19 @@ public class Broadcaster {
}
/**
- * Broadcast an event out
+ * Announce an event out to peer kylin servers
*/
- public void queue(String entity, String event, String key) {
+ public void announce(String entity, String event, String key) {
+ announce(new BroadcastEvent(entity, event, key));
+ }
+
+ public void announce(BroadcastEvent event) {
if (broadcastEvents == null)
return;
try {
counter.incrementAndGet();
- broadcastEvents.putLast(new BroadcastEvent(entity, event, key));
+ broadcastEvents.putLast(event);
} catch (Exception e) {
counter.decrementAndGet();
logger.error("error putting BroadcastEvent", e);
@@ -282,6 +305,40 @@ public class Broadcaster {
return counter.getAndSet(0);
}
+ // ============================================================================
+
+ public static class DefaultSyncErrorHandler implements SyncErrorHandler {
+ Broadcaster broadcaster;
+ int maxRetryTimes;
+
+ @Override
+ public void init(Broadcaster broadcaster) {
+ this.maxRetryTimes = broadcaster.getConfig().getCacheSyncRetrys();
+ this.broadcaster = broadcaster;
+ }
+
+ @Override
+ public void handleAnnounceError(String targetNode, RestClient restClient, BroadcastEvent event) {
+ int retry = event.getRetryTime() + 1;
+
+ // when sync failed, put back to queue to retry
+ if (retry < maxRetryTimes) {
+ event.setRetryTime(retry);
+ event.setTargetNode(targetNode);
+ broadcaster.announce(event);
+ } else {
+ logger.error("Announce broadcast event exceeds retry limit, abandon targetNode {} broadcastEvent {}",
+ targetNode, event);
+ }
+ }
+ }
+
+ public interface SyncErrorHandler {
+ void init(Broadcaster broadcaster);
+
+ void handleAnnounceError(String targetNode, RestClient restClient, BroadcastEvent event);
+ }
+
public enum Event {
CREATE("create"), UPDATE("update"), DROP("drop");
@@ -326,6 +383,8 @@ public class Broadcaster {
public static class BroadcastEvent {
private int retryTime;
+ private String targetNode; // NULL means to all
+
private String entity;
private String event;
private String cacheKey;
@@ -345,6 +404,14 @@ public class Broadcaster {
this.retryTime = retryTime;
}
+ public String getTargetNode() {
+ return targetNode;
+ }
+
+ public void setTargetNode(String targetNode) {
+ this.targetNode = targetNode;
+ }
+
public String getEntity() {
return entity;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/baf7133d/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/SingleValueCache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/SingleValueCache.java
index 4bfaeae..f803c8b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/SingleValueCache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/SingleValueCache.java
@@ -49,9 +49,9 @@ public abstract class SingleValueCache<K, V> extends AbstractCache<K, V> {
innerCache.put(key, value);
if (!exists) {
- getBroadcaster().queue(syncEntity, Broadcaster.Event.CREATE.getType(), key.toString());
+ getBroadcaster().announce(syncEntity, Broadcaster.Event.CREATE.getType(), key.toString());
} else {
- getBroadcaster().queue(syncEntity, Broadcaster.Event.UPDATE.getType(), key.toString());
+ getBroadcaster().announce(syncEntity, Broadcaster.Event.UPDATE.getType(), key.toString());
}
}
@@ -65,7 +65,7 @@ public abstract class SingleValueCache<K, V> extends AbstractCache<K, V> {
innerCache.remove(key);
if (exists) {
- getBroadcaster().queue(syncEntity, Broadcaster.Event.DROP.getType(), key.toString());
+ getBroadcaster().announce(syncEntity, Broadcaster.Event.DROP.getType(), key.toString());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/baf7133d/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/BroadcasterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/BroadcasterTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/BroadcasterTest.java
index 80f26f9..762512f 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/BroadcasterTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/BroadcasterTest.java
@@ -21,9 +21,12 @@ package org.apache.kylin.metadata.cachesync;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.cachesync.Broadcaster.BroadcastEvent;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.metadata.cachesync.Broadcaster.Listener;
+import org.apache.kylin.metadata.cachesync.Broadcaster.SyncErrorHandler;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -64,6 +67,7 @@ public class BroadcasterTest extends LocalFileMetadataTestCase {
broadcaster.notifyListener("test", Event.UPDATE, "");
+ broadcaster.stopAnnounce();
Broadcaster.staticListenerMap.clear();
}
@@ -90,6 +94,45 @@ public class BroadcasterTest extends LocalFileMetadataTestCase {
broadcaster.notifyNonStaticListener("test", Event.UPDATE, "");
+ broadcaster.stopAnnounce();
Broadcaster.staticListenerMap.clear();
}
+
+ @Test
+ public void testAnnounceErrorHandler() throws IOException, InterruptedException {
+ System.setProperty("kylin.server.cluster-servers", "localhost:717");
+ System.setProperty("kylin.metadata.sync-error-handler", MockupErrHandler.class.getName());
+ try {
+ Broadcaster broadcaster = Broadcaster.getInstance(getTestConfig());
+
+ broadcaster.announce("all", "update", "all");
+
+ for (int i = 0; i < 30 && MockupErrHandler.atom.get() == 0; i++) {
+ Thread.sleep(1000);
+ }
+
+ broadcaster.stopAnnounce();
+ Broadcaster.staticListenerMap.clear();
+ } finally {
+ System.clearProperty("kylin.server.cluster-servers");
+ System.clearProperty("kylin.metadata.sync-error-handler");
+ }
+
+ Assert.assertTrue(MockupErrHandler.atom.get() > 0);
+ }
+
+ public static class MockupErrHandler implements SyncErrorHandler {
+ static AtomicInteger atom = new AtomicInteger();
+
+ @Override
+ public void init(Broadcaster broadcaster) {
+ }
+
+ @Override
+ public void handleAnnounceError(String targetNode, RestClient restClient, BroadcastEvent event) {
+ Assert.assertEquals("localhost:717", targetNode);
+ atom.incrementAndGet();
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/baf7133d/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
index 14052ce..467ef82 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java
@@ -53,7 +53,7 @@ public class InitialTaskManager implements InitializingBean {
for (String taskClass : taskClasses) {
try {
InitialTask task = (InitialTask) Class.forName(taskClass).newInstance();
- logger.info("Running task: " + taskClass);
+ logger.info("Running initial task: " + taskClass);
task.execute();
} catch (Throwable e) {
logger.error("Initial task failed: " + taskClass, e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/baf7133d/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 98e06e0..b61309e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
@@ -49,6 +50,7 @@ public class CacheService extends BasicService implements InitializingBean {
@Override
public void onClearAll(Broadcaster broadcaster) throws IOException {
cleanAllDataCache();
+ HBaseConnection.clearConnCache(); // take the chance to clear HBase connection cache as well
}
@Override
@@ -104,7 +106,7 @@ public class CacheService extends BasicService implements InitializingBean {
public void annouceWipeCache(String entity, String event, String cacheKey) {
Broadcaster broadcaster = Broadcaster.getInstance(getConfig());
- broadcaster.queue(entity, event, cacheKey);
+ broadcaster.announce(entity, event, cacheKey);
}
public void notifyMetadataChange(String entity, Event event, String cacheKey) throws IOException {
[33/50] [abbrv] kylin git commit: KYLIN-2909 use EmailTemplateFactory
in CubingJob (Thanks Yanghong Zhong and Julian Pan)
Posted by li...@apache.org.
KYLIN-2909 use EmailTemplateFactory in CubingJob (Thanks Yanghong Zhong and Julian Pan)
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c96e1b19
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c96e1b19
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c96e1b19
Branch: refs/heads/sync
Commit: c96e1b1947ca8764066ff4390ea1f9e4aa0f6c40
Parents: 7dc1875
Author: Zhong <nj...@apache.org>
Authored: Sat Sep 30 14:26:24 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sun Jan 28 15:57:30 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 +
.../kylin/common/util/EmailTemplateFactory.java | 11 +-
.../main/resources/templates/JOB_DISCARD.ftl | 2 +-
.../main/resources/templates/JOB_SUCCEED.ftl | 2 +-
.../common/util/EmailTemplateFactoryTest.java | 32 +++++
.../kylin/job/constant/ExecutableConstants.java | 1 -
.../kylin/job/execution/AbstractExecutable.java | 92 ++++++++++---
.../kylin/job/util/ExecutableStateUtil.java | 38 ++++++
.../org/apache/kylin/engine/mr/CubingJob.java | 134 ++++++++-----------
9 files changed, 202 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c96e1b19/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 6d1b7f9..7b48935 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -661,6 +661,10 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.job.tracking-url-pattern", "");
}
+ public int getJobMetadataPersistRetry() {
+ return Integer.parseInt(this.getOptional("kylin.job.metadata-persist-retry", "5"));
+ }
+
// ============================================================================
// SOURCE.HIVE
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/c96e1b19/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java b/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java
index fcf554d..2acea5d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java
@@ -27,6 +27,8 @@ import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Joiner;
+
import freemarker.template.Configuration;
import freemarker.template.Template;
@@ -50,14 +52,7 @@ public class EmailTemplateFactory {
}
public static String getEmailTitle(String... titleParts) {
- StringBuilder sb = new StringBuilder();
- for (String part : titleParts) {
- if (sb.length() > 0) {
- sb.append("-");
- }
- sb.append("[" + part + "]");
- }
- return sb.toString();
+ return "[" + Joiner.on("]-[").join(titleParts).toString() + "]";
}
private static EmailTemplateFactory instance = new EmailTemplateFactory();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c96e1b19/core-common/src/main/resources/templates/JOB_DISCARD.ftl
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/templates/JOB_DISCARD.ftl b/core-common/src/main/resources/templates/JOB_DISCARD.ftl
index fbef3f7..b00e9aa 100644
--- a/core-common/src/main/resources/templates/JOB_DISCARD.ftl
+++ b/core-common/src/main/resources/templates/JOB_DISCARD.ftl
@@ -33,7 +33,7 @@
<span style="
line-height: 1.1;font-size: 18px;">
<p style="text-align:left;">Dear Kylin user,</p>
- <p>It's a pity that the job is discarded.Thank you for using Kylin.</p>
+ <p>It's a pity that the job is discarded. Thank you for using Kylin.</p>
</span>
<hr style="margin-top: 20px;
margin-bottom: 20px;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c96e1b19/core-common/src/main/resources/templates/JOB_SUCCEED.ftl
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/templates/JOB_SUCCEED.ftl b/core-common/src/main/resources/templates/JOB_SUCCEED.ftl
index 4b443d5..f1fb45c 100644
--- a/core-common/src/main/resources/templates/JOB_SUCCEED.ftl
+++ b/core-common/src/main/resources/templates/JOB_SUCCEED.ftl
@@ -32,7 +32,7 @@
<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
<span style="line-height: 1.1;font-size: 18px;">
<p style="text-align:left;">Dear Kylin user,</p>
- <p>Congratulations! Please feel free to query based on kylin cube.Thank you for using Kylin.</p>
+ <p>Congratulations! Please feel free to query based on kylin cube. Thank you for using Kylin.</p>
</span>
<hr style="margin-top: 20px;
margin-bottom: 20px;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c96e1b19/core-common/src/test/java/org/apache/kylin/common/util/EmailTemplateFactoryTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/EmailTemplateFactoryTest.java b/core-common/src/test/java/org/apache/kylin/common/util/EmailTemplateFactoryTest.java
new file mode 100644
index 0000000..6acbd75
--- /dev/null
+++ b/core-common/src/test/java/org/apache/kylin/common/util/EmailTemplateFactoryTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class EmailTemplateFactoryTest {
+
+ @Test
+ public void testGetEmailTitle() {
+ String[] titleParts = new String[] { "JOB", "SUCCEED" };
+ Assert.assertEquals("[" + titleParts[0] + "]-[" + titleParts[1] + "]",
+ EmailTemplateFactory.getEmailTitle(titleParts));
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c96e1b19/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
index cf61722..b9a3651 100644
--- a/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
+++ b/core-job/src/main/java/org/apache/kylin/job/constant/ExecutableConstants.java
@@ -62,5 +62,4 @@ public final class ExecutableConstants {
public static final String STEP_NAME_GARBAGE_COLLECTION_HBASE = "Garbage Collection on HBase";
public static final String STEP_NAME_GARBAGE_COLLECTION_HDFS = "Garbage Collection on HDFS";
public static final String STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE = "Redistribute Flat Hive Table";
- public static final String NOTIFY_EMAIL_TEMPLATE = "<div><b>Build Result of Job ${job_name}</b><pre><ul>" + "<li>Build Result: <b>${result}</b></li>" + "<li>Job Engine: ${job_engine}</li>" + "<li>Env: ${env_name}</li>" + "<li>Project: ${project_name}</li>" + "<li>Cube Name: ${cube_name}</li>" + "<li>Source Records Count: ${source_records_count}</li>" + "<li>Start Time: ${start_time}</li>" + "<li>Duration: ${duration}</li>" + "<li>MR Waiting: ${mr_waiting}</li>" + "<li>Last Update Time: ${last_update_time}</li>" + "<li>Submitter: ${submitter}</li>" + "<li>Error Log: ${error_log}</li>" + "</ul></pre><div/>";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c96e1b19/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 6a0db97..f4015bf 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -24,12 +24,16 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.regex.Matcher;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.EmailTemplateEnum;
+import org.apache.kylin.common.util.EmailTemplateFactory;
import org.apache.kylin.common.util.MailService;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.exception.PersistentException;
import org.apache.kylin.job.impl.threadpool.DefaultContext;
@@ -64,16 +68,16 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
public AbstractExecutable() {
setId(UUID.randomUUID().toString());
}
-
+
protected void initConfig(KylinConfig config) {
Preconditions.checkState(this.config == null || this.config == config);
this.config = config;
}
-
+
protected KylinConfig getConfig() {
return config;
}
-
+
protected ExecutableManager getManager() {
return ExecutableManager.getInstance(config);
}
@@ -84,6 +88,36 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
getManager().updateJobOutput(getId(), ExecutableState.RUNNING, info, null);
}
+ private void onExecuteFinishedWithRetry(ExecuteResult result, ExecutableContext executableContext)
+ throws ExecuteException {
+ Throwable exception;
+ int nRetry = 0;
+ do {
+ nRetry++;
+ exception = null;
+ try {
+ onExecuteFinished(result, executableContext);
+ } catch (Exception e) {
+ logger.error(nRetry + "th retries for onExecuteFinished fails due to {}", e);
+ if (isMetaDataPersistException(e)) {
+ exception = e;
+ try {
+ Thread.sleep(1000L * (long) Math.pow(4, nRetry));
+ } catch (InterruptedException exp) {
+ throw new RuntimeException(exp);
+ }
+ } else {
+ throw e;
+ }
+ }
+ } while (exception != null && nRetry <= executableContext.getConfig().getJobMetadataPersistRetry());
+
+ if (exception != null) {
+ handleMetadataPersistException(executableContext, exception);
+ throw new ExecuteException(exception);
+ }
+ }
+
protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) {
setEndTime(System.currentTimeMillis());
if (!isDiscarded() && !isRunnable()) {
@@ -134,28 +168,42 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
retry++;
} while (needRetry(result, exception));
- //check exception in result to avoid retry on ChainedExecutable(only need retry on subtask actually)
- if (exception != null || result.getThrowable() != null) {
+ if (exception != null) {
onExecuteError(exception, executableContext);
throw new ExecuteException(exception);
}
- onExecuteFinished(result, executableContext);
+ onExecuteFinishedWithRetry(result, executableContext);
+ } catch (ExecuteException e) {
+ throw e;
} catch (Exception e) {
- if (isMetaDataPersistException(e)) {
- handleMetaDataPersistException(e);
- }
- if (e instanceof ExecuteException) {
- throw e;
- } else {
- throw new ExecuteException(e);
- }
+ throw new ExecuteException(e);
}
return result;
}
- protected void handleMetaDataPersistException(Exception e) {
- // do nothing.
+ protected void handleMetadataPersistException(ExecutableContext context, Throwable exception) {
+ final String[] adminDls = context.getConfig().getAdminDls();
+ if (adminDls == null || adminDls.length < 1) {
+ logger.warn("no need to send email, user list is empty");
+ return;
+ }
+ List<String> users = Lists.newArrayList(adminDls);
+
+ Map<String, Object> dataMap = Maps.newHashMap();
+ dataMap.put("job_name", getName());
+ dataMap.put("env_name", context.getConfig().getDeployEnv());
+ dataMap.put("submitter", StringUtil.noBlank(getSubmitter(), "missing submitter"));
+ dataMap.put("job_engine", EmailTemplateFactory.getLocalHostName());
+ dataMap.put("error_log",
+ Matcher.quoteReplacement(StringUtil.noBlank(exception.getMessage(), "no error message")));
+
+ String content = EmailTemplateFactory.getInstance().buildEmailContent(EmailTemplateEnum.METADATA_PERSIST_FAIL,
+ dataMap);
+ String title = EmailTemplateFactory.getEmailTitle("METADATA PERSIST", "FAIL",
+ context.getConfig().getDeployEnv());
+
+ new MailService(context.getConfig()).sendMail(users, title, content);
}
private boolean isMetaDataPersistException(Exception e) {
@@ -299,7 +347,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
logger.info("job name:" + getName());
logger.info("submitter:" + getSubmitter());
logger.info("notify list:" + users);
- new MailService(kylinConfig).sendMail(users, email.getLeft(), email.getRight());
+ new MailService(kylinConfig).sendMail(users, email.getFirst(), email.getSecond());
}
protected void sendMail(Pair<String, String> email) {
@@ -335,7 +383,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
public static long getEndTime(Output output) {
return getExtraInfoAsLong(output, END_TIME, 0L);
}
-
+
public static long getInterruptTime(Output output) {
return getExtraInfoAsLong(output, INTERRUPT_TIME, 0L);
}
@@ -437,8 +485,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
if (retryableEx == null || retryableEx.length == 0) {
return true;
}
- if ((result != null && isRetryableExecutionResult(result))
- || e != null && isRetrableException(e)) {
+ if ((result != null && isRetryableExecutionResult(result)) || e != null && isRetrableException(e)) {
return true;
}
return false;
@@ -446,6 +493,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
@Override
public String toString() {
- return Objects.toStringHelper(this).add("id", getId()).add("name", getName()).add("state", getStatus()).toString();
+ return Objects.toStringHelper(this).add("id", getId()).add("name", getName()).add("state", getStatus())
+ .toString();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c96e1b19/core-job/src/main/java/org/apache/kylin/job/util/ExecutableStateUtil.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/util/ExecutableStateUtil.java b/core-job/src/main/java/org/apache/kylin/job/util/ExecutableStateUtil.java
new file mode 100644
index 0000000..66f806c
--- /dev/null
+++ b/core-job/src/main/java/org/apache/kylin/job/util/ExecutableStateUtil.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.util;
+
+import org.apache.kylin.common.util.EmailTemplateEnum;
+import org.apache.kylin.job.execution.ExecutableState;
+
+public class ExecutableStateUtil {
+
+ public static EmailTemplateEnum getEmailTemplateEnum(ExecutableState state) {
+ switch (state) {
+ case ERROR:
+ return EmailTemplateEnum.JOB_ERROR;
+ case DISCARDED:
+ return EmailTemplateEnum.JOB_DISCARD;
+ case SUCCEED:
+ return EmailTemplateEnum.JOB_SUCCEED;
+ default:
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c96e1b19/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 6f26c35..548dcb9 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -18,20 +18,20 @@
package org.apache.kylin.engine.mr;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
+import java.util.Map;
import java.util.TimeZone;
import java.util.regex.Matcher;
-import org.apache.commons.lang3.tuple.Pair;
+
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.EmailTemplateEnum;
+import org.apache.kylin.common.util.EmailTemplateFactory;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -47,12 +47,15 @@ import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.job.execution.Output;
import org.apache.kylin.job.metrics.JobMetricsFacade;
+import org.apache.kylin.job.util.ExecutableStateUtil;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
/**
*/
@@ -195,52 +198,61 @@ public class CubingJob extends DefaultChainedExecutable {
CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig())
.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
final Output output = getManager().getOutput(getId());
- String logMsg;
state = output.getState();
if (state != ExecutableState.ERROR
&& !cubeInstance.getDescriptor().getStatusNeedNotify().contains(state.toString())) {
logger.info("state:" + state + " no need to notify users");
return null;
}
- switch (state) {
- case ERROR:
- logMsg = output.getVerboseMsg();
- break;
- case DISCARDED:
- logMsg = "job has been discarded";
- break;
- case SUCCEED:
- logMsg = "job has succeeded";
- break;
- default:
+
+ EmailTemplateEnum templateEnum = ExecutableStateUtil.getEmailTemplateEnum(state);
+ if (templateEnum == null) {
+ logger.info("Cannot find email template for job state: " + state);
return null;
}
- String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE;
- content = content.replaceAll("\\$\\{job_name\\}", getName());
- content = content.replaceAll("\\$\\{result\\}", state.toString());
- content = content.replaceAll("\\$\\{env_name\\}", getDeployEnvName());
- content = content.replaceAll("\\$\\{project_name\\}", getProjectName());
- content = content.replaceAll("\\$\\{cube_name\\}", CubingExecutableUtil.getCubeName(this.getParams()));
- content = content.replaceAll("\\$\\{source_records_count\\}", String.valueOf(findSourceRecordCount()));
- content = content.replaceAll("\\$\\{start_time\\}", new Date(getStartTime()).toString());
- content = content.replaceAll("\\$\\{duration\\}", getDuration() / 60000 + "mins");
- content = content.replaceAll("\\$\\{mr_waiting\\}", getMapReduceWaitTime() / 60000 + "mins");
- content = content.replaceAll("\\$\\{last_update_time\\}", new Date(getLastModified()).toString());
- content = content.replaceAll("\\$\\{submitter\\}", StringUtil.noBlank(getSubmitter(), "missing submitter"));
- content = content.replaceAll("\\$\\{error_log\\}",
- Matcher.quoteReplacement(StringUtil.noBlank(logMsg, "no error message")));
-
- try {
- InetAddress inetAddress = InetAddress.getLocalHost();
- content = content.replaceAll("\\$\\{job_engine\\}", inetAddress.getCanonicalHostName());
- } catch (UnknownHostException e) {
- logger.warn(e.getLocalizedMessage(), e);
- }
- String title = "[" + state.toString() + "] - [" + getDeployEnvName() + "] - [" + getProjectName() + "] - "
- + CubingExecutableUtil.getCubeName(this.getParams());
+ Map<String, Object> dataMap = Maps.newHashMap();
+ dataMap.put("job_name", getName());
+ dataMap.put("env_name", getDeployEnvName());
+ dataMap.put("submitter", StringUtil.noBlank(getSubmitter(), "missing submitter"));
+ dataMap.put("job_engine", EmailTemplateFactory.getLocalHostName());
+ dataMap.put("project_name", getProjectName());
+ dataMap.put("cube_name", cubeInstance.getName());
+ dataMap.put("source_records_count", String.valueOf(findSourceRecordCount()));
+ dataMap.put("start_time", new Date(getStartTime()).toString());
+ dataMap.put("duration", getDuration() / 60000 + "mins");
+ dataMap.put("mr_waiting", getMapReduceWaitTime() / 60000 + "mins");
+ dataMap.put("last_update_time", new Date(getLastModified()).toString());
+
+ if (state == ExecutableState.ERROR) {
+ AbstractExecutable errorTask = null;
+ Output errorOutput = null;
+ for (AbstractExecutable task : getTasks()) {
+ errorOutput = getManager().getOutput(task.getId());
+ if (errorOutput.getState() == ExecutableState.ERROR) {
+ errorTask = task;
+ break;
+ }
+ }
+ Preconditions.checkNotNull(errorTask,
+ "None of the sub tasks of cubing job " + getId() + " is error and this job should become success.");
- return Pair.of(title, content);
+ dataMap.put("error_step", errorTask.getName());
+ if (errorTask instanceof MapReduceExecutable) {
+ final String mrJobId = errorOutput.getExtra().get(ExecutableConstants.MR_JOB_ID);
+ dataMap.put("mr_job_id", StringUtil.noBlank(mrJobId, "Not initialized"));
+ } else {
+ dataMap.put("mr_job_id", EmailTemplateFactory.NA);
+ }
+ dataMap.put("error_log",
+ Matcher.quoteReplacement(StringUtil.noBlank(output.getVerboseMsg(), "no error message")));
+ }
+
+ String content = EmailTemplateFactory.getInstance()
+ .buildEmailContent(ExecutableStateUtil.getEmailTemplateEnum(state), dataMap);
+ String title = EmailTemplateFactory.getEmailTitle("JOB", state.toString(), getDeployEnvName(), getProjectName(),
+ cubeInstance.getName());
+ return Pair.newPair(title, content);
}
@Override
@@ -297,46 +309,6 @@ public class CubingJob extends DefaultChainedExecutable {
return timeCost * 1.0 / size;
}
- /**
- * build fail because the metadata store has problem.
- * @param exception
- */
- @Override
- protected void handleMetaDataPersistException(Exception exception) {
- String title = "[ERROR] - [" + getDeployEnvName() + "] - [" + getProjectName() + "] - "
- + CubingExecutableUtil.getCubeName(this.getParams());
- String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE;
- final String UNKNOWN = "UNKNOWN";
- String errMsg = null;
- if (exception != null) {
- final StringWriter out = new StringWriter();
- exception.printStackTrace(new PrintWriter(out));
- errMsg = out.toString();
- }
-
- content = content.replaceAll("\\$\\{job_name\\}", getName());
- content = content.replaceAll("\\$\\{result\\}", ExecutableState.ERROR.toString());
- content = content.replaceAll("\\$\\{env_name\\}", getDeployEnvName());
- content = content.replaceAll("\\$\\{project_name\\}", getProjectName());
- content = content.replaceAll("\\$\\{cube_name\\}", CubingExecutableUtil.getCubeName(this.getParams()));
- content = content.replaceAll("\\$\\{source_records_count\\}", UNKNOWN);
- content = content.replaceAll("\\$\\{start_time\\}", UNKNOWN);
- content = content.replaceAll("\\$\\{duration\\}", UNKNOWN);
- content = content.replaceAll("\\$\\{mr_waiting\\}", UNKNOWN);
- content = content.replaceAll("\\$\\{last_update_time\\}", UNKNOWN);
- content = content.replaceAll("\\$\\{submitter\\}", StringUtil.noBlank(getSubmitter(), "missing submitter"));
- content = content.replaceAll("\\$\\{error_log\\}",
- Matcher.quoteReplacement(StringUtil.noBlank(errMsg, "no error message")));
-
- try {
- InetAddress inetAddress = InetAddress.getLocalHost();
- content = content.replaceAll("\\$\\{job_engine\\}", inetAddress.getCanonicalHostName());
- } catch (UnknownHostException e) {
- logger.warn(e.getLocalizedMessage(), e);
- }
- sendMail(Pair.of(title, content));
- }
-
public long getMapReduceWaitTime() {
return getExtraInfoAsLong(MAP_REDUCE_WAIT_TIME, 0L);
}
[10/50] [abbrv] kylin git commit: KYLIN-3173,
reset DefaultScheduler's initialized/hasStarted when shutdown.
Posted by li...@apache.org.
KYLIN-3173, reset DefaultScheduler's initialized/hasStarted when shutdown.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/195fdef5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/195fdef5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/195fdef5
Branch: refs/heads/sync
Commit: 195fdef501ee03f5e31fdae81d282f3e0cf1ee0f
Parents: a31bce6
Author: tttMelody <24...@qq.com>
Authored: Wed Jan 17 14:48:18 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/impl/threadpool/DefaultScheduler.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/195fdef5/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
index 6af0192..0b01284 100644
--- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
+++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DefaultScheduler.java
@@ -366,6 +366,8 @@ public class DefaultScheduler implements Scheduler<AbstractExecutable>, Connecti
public void shutdown() throws SchedulerException {
logger.info("Shutting down DefaultScheduler ....");
jobLock.unlockJobEngine();
+ initialized = false;
+ hasStarted = false;
try {
fetcherPool.shutdownNow();//interrupt
fetcherPool.awaitTermination(1, TimeUnit.MINUTES);
[19/50] [abbrv] kylin git commit: minor,
add switch table-dir-create-first for build cube with hive step.
Posted by li...@apache.org.
minor, add switch table-dir-create-first for build cube with hive step.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ce1775a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ce1775a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ce1775a5
Branch: refs/heads/sync
Commit: ce1775a5ab0e775bce6685e7801f114a4aff9b9b
Parents: 543124d
Author: yiming.xu <10...@qq.com>
Authored: Wed Jan 24 17:33:18 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../main/java/org/apache/kylin/source/hive/HiveMRInput.java | 8 +++++---
.../java/org/apache/kylin/source/hive/HiveMRInputTest.java | 2 +-
3 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5045b2f..6d1b7f9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -755,6 +755,10 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.source.hive.sparksql-beeline-params", "");
}
+ public boolean getHiveTableDirCreateFirst() {
+ return Boolean.parseBoolean(getOptional("kylin.source.hive.table-dir-create-first", "false"));
+ }
+
public String getFlatHiveTableClusterByDictColumn() {
return getOptional("kylin.source.hive.flat-table-cluster-by-dict-column");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 31a99cd..f02d2ea 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -179,9 +179,11 @@ public class HiveMRInput implements IMRInput {
protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) {
String jobWorkingDir = JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
- // Create work dir to avoid hive create it,
- // the difference is that the owners are different.
- checkAndCreateWorkDir(jobWorkingDir);
+ if (KylinConfig.getInstanceFromEnv().getHiveTableDirCreateFirst()) {
+ // Create work dir to avoid hive create it,
+ // the difference is that the owners are different.
+ checkAndCreateWorkDir(jobWorkingDir);
+ }
return jobWorkingDir;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ce1775a5/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
index 6d0737d..f33ee74 100644
--- a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -41,8 +41,8 @@ public class HiveMRInputTest {
try {
KylinConfig kylinConfig = mock(KylinConfig.class);
KylinConfig.setKylinConfigThreadLocal(kylinConfig);
+ when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true);
when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/");
-
DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class);
defaultChainedExecutable.setId(UUID.randomUUID().toString());
[30/50] [abbrv] kylin git commit: KYLIN-2760, remove useless ACL code.
Posted by li...@apache.org.
KYLIN-2760, remove useless ACL code.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d2203f43
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d2203f43
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d2203f43
Branch: refs/heads/sync
Commit: d2203f430f4f9dcf0bd681b7657f31906a32a5bc
Parents: 95e3d07
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 15 17:40:47 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/service/AccessService.java | 1 +
.../org/apache/kylin/rest/service/CubeService.java | 12 ------------
.../org/apache/kylin/rest/service/JobService.java | 17 -----------------
.../apache/kylin/rest/service/ModelService.java | 13 -------------
4 files changed, 1 insertion(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d2203f43/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 3732c15..8498ecd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -218,6 +218,7 @@ public class AccessService {
return acl;
}
+ @Deprecated
@Transactional
public void inherit(AclEntity ae, AclEntity parentAe) {
Message msg = MsgPicker.getMsg();
http://git-wip-us.apache.org/repos/asf/kylin/blob/d2203f43/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 154354d..8238f5c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -71,7 +71,6 @@ import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.response.MetricsResponse;
-import org.apache.kylin.rest.security.AclPermission;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.ValidateUtil;
import org.slf4j.Logger;
@@ -103,10 +102,6 @@ public class CubeService extends BasicService implements InitializingBean {
protected Cache<String, HBaseResponse> htableInfoCache = CacheBuilder.newBuilder().build();
@Autowired
- @Qualifier("accessService")
- private AccessService accessService;
-
- @Autowired
@Qualifier("projectService")
private ProjectService projectService;
@@ -216,10 +211,6 @@ public class CubeService extends BasicService implements InitializingBean {
logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids");
createdCube = getCubeManager().createCube(cubeName, project.getName(), createdDesc, owner);
- accessService.init(createdCube, AclPermission.ADMINISTRATION);
-
- accessService.inherit(createdCube, project);
-
return createdCube;
}
@@ -289,8 +280,6 @@ public class CubeService extends BasicService implements InitializingBean {
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
ProjectInstance newProject = projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(),
newProjectName, owner);
-
- accessService.inherit(cube, newProject);
}
return updatedCubeDesc;
@@ -315,7 +304,6 @@ public class CubeService extends BasicService implements InitializingBean {
int cubeNum = getCubeManager().getCubesByDesc(cube.getDescriptor().getName()).size();
getCubeManager().dropCube(cube.getName(), cubeNum == 1);//only delete cube desc when no other cube is using it
- accessService.clean(cube, true);
}
/**
http://git-wip-us.apache.org/repos/asf/kylin/blob/d2203f43/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index bd90742..e447031 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -74,7 +74,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
@@ -97,10 +96,6 @@ public class JobService extends BasicService implements InitializingBean {
private static final Logger logger = LoggerFactory.getLogger(JobService.class);
@Autowired
- @Qualifier("accessService")
- private AccessService accessService;
-
- @Autowired
private AclEvaluate aclEvaluate;
/*
@@ -213,9 +208,6 @@ public class JobService extends BasicService implements InitializingBean {
JobInstance jobInstance = submitJobInternal(cube, tsRange, segRange, sourcePartitionOffsetStart,
sourcePartitionOffsetEnd, buildType, force, submitter);
- accessService.init(jobInstance, null);
- accessService.inherit(jobInstance, cube);
-
return jobInstance;
}
@@ -282,13 +274,6 @@ public class JobService extends BasicService implements InitializingBean {
String submitter) throws IOException, JobException {
Pair<JobInstance, List<JobInstance>> result = submitOptimizeJobInternal(cube, cuboidsRecommend, submitter);
- accessService.init(result.getFirst(), null);
- accessService.inherit(result.getFirst(), cube);
- for (JobInstance jobInstance : result.getSecond()) {
- accessService.init(jobInstance, null);
- accessService.inherit(jobInstance, cube);
- }
-
return result;
}
@@ -391,8 +376,6 @@ public class JobService extends BasicService implements InitializingBean {
getExecutableManager().addJob(optimizeJob);
JobInstance optimizeJobInstance = getSingleJobInstance(optimizeJob);
- accessService.init(optimizeJobInstance, null);
- accessService.inherit(optimizeJobInstance, cubeInstance);
/** Update the checkpoint job */
checkpointExecutable.getSubTasksForCheck().set(checkpointExecutable.getSubTasksForCheck().indexOf(toBeReplaced),
http://git-wip-us.apache.org/repos/asf/kylin/blob/d2203f43/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index ee01f3e..463107b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -42,7 +42,6 @@ import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
-import org.apache.kylin.rest.security.AclPermission;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.ValidateUtil;
import org.slf4j.Logger;
@@ -65,10 +64,6 @@ public class ModelService extends BasicService {
private static final Logger logger = LoggerFactory.getLogger(ModelService.class);
@Autowired
- @Qualifier("accessService")
- private AccessService accessService;
-
- @Autowired
@Qualifier("cubeMgmtService")
private CubeService cubeService;
@@ -142,12 +137,6 @@ public class ModelService extends BasicService {
DataModelDesc createdDesc = null;
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
createdDesc = getDataModelManager().createDataModelDesc(desc, projectName, owner);
-
- if (!desc.isDraft()) {
- accessService.init(createdDesc, AclPermission.ADMINISTRATION);
- ProjectInstance project = getProjectManager().getProject(projectName);
- accessService.inherit(createdDesc, project);
- }
return createdDesc;
}
@@ -169,8 +158,6 @@ public class ModelService extends BasicService {
}
getDataModelManager().dropModel(desc);
-
- accessService.clean(desc, true);
}
public boolean isTableInAnyModel(TableDesc table) {
[44/50] [abbrv] kylin git commit: Ignore SonarCloud for external PR
Posted by li...@apache.org.
Ignore SonarCloud for external PR
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ff8bece2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ff8bece2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ff8bece2
Branch: refs/heads/sync
Commit: ff8bece2114dc6c2a401b0b774ad14d530fd8c65
Parents: 6a2ee80
Author: Billy Liu <bi...@apache.org>
Authored: Wed Jan 31 11:27:26 2018 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Jan 31 11:45:31 2018 +0800
----------------------------------------------------------------------
.travis.yml | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff8bece2/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index cbf3515..ce76775 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,16 +16,23 @@
# limitations under the License.
#
language: java
+
addons:
sonarcloud:
organization: "kylin"
token:
secure: "c918989d018ae3af899798716c35f4fe125296a4"
+
jdk:
- oraclejdk8
script:
- - mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent test sonar:sonar
+ - mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent test
+ - if [[ -n "${TRAVIS_PULL_REQUEST_SLUG}" && "${TRAVIS_PULL_REQUEST_SLUG}" != "${TRAVIS_REPO_SLUG}" ]]; then
+ echo "The pull request from ${TRAVIS_PULL_REQUEST_SLUG} is an EXTERNAL pull request. Skip sonar analysis.";
+ else
+ mvn sonar:sonar;
+ fi
notification:
email:
@@ -33,3 +40,6 @@ notification:
- dev@kylin.apache.org
on_failure: always
+after_success:
+ - bash <(curl -s https://codecov.io/bash)
+
[31/50] [abbrv] kylin git commit: KYLIN-3090,
complete UUID if entity hasn't.
Posted by li...@apache.org.
KYLIN-3090, complete UUID if entity hasn't.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/95e3d077
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/95e3d077
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/95e3d077
Branch: refs/heads/sync
Commit: 95e3d077cb69b2e8086b4646c173e0098f44f265
Parents: adf645e
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 15 18:48:57 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../metadata/cachesync/CachedCrudAssist.java | 8 ++-
.../cachesync/CachedCrudAssistTest.java | 63 ++++++++++++++++++++
2 files changed, 70 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/95e3d077/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
index c7244fe..b3c200e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/CachedCrudAssist.java
@@ -173,7 +173,7 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> {
public T save(T entity) throws IOException {
Preconditions.checkArgument(entity != null);
- Preconditions.checkArgument(entity.getUuid() != null);
+ completeUuidIfNeeded(entity);
Preconditions.checkArgument(entityType.isInstance(entity));
String resName = entity.resourceName();
@@ -199,6 +199,12 @@ abstract public class CachedCrudAssist<T extends RootPersistentEntity> {
return reload(resName);
}
+ private void completeUuidIfNeeded(T entity) {
+ if (entity.getUuid() == null) {
+ entity.updateRandomUuid();
+ }
+ }
+
public void delete(T entity) throws IOException {
delete(entity.resourceName());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/95e3d077/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java
new file mode 100644
index 0000000..482ec09
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/cachesync/CachedCrudAssistTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.cachesync;
+
+import java.io.IOException;
+
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CachedCrudAssistTest extends LocalFileMetadataTestCase {
+ @Before
+ public void setup() throws Exception {
+ staticCreateTestMetadata();
+ }
+
+ @Test
+ public void testSave() throws IOException {
+ CachedCrudAssist<TestEntity> crudAssist = new CachedCrudAssist<TestEntity>(getStore() //
+ , "/test" //
+ , TestEntity.class //
+ , new CaseInsensitiveStringCache<TestEntity>(getTestConfig(), "test")) { //
+ @Override
+ protected TestEntity initEntityAfterReload(TestEntity entity, String resourceName) {
+ return entity;
+ }
+ };
+
+ TestEntity entity = new TestEntity();
+ Assert.assertNull(entity.getUuid());
+ crudAssist.save(entity);
+ Assert.assertNotNull(entity.getUuid());
+ }
+
+ @After
+ public void after() throws Exception {
+ cleanAfterClass();
+ }
+}
+
+class TestEntity extends RootPersistentEntity {
+ public TestEntity() {
+ }
+}
\ No newline at end of file
[03/50] [abbrv] kylin git commit: minor,
create job work dir with Materialize Hive View in Lookup Tables job.
Posted by li...@apache.org.
minor, create job work dir with Materialize Hive View in Lookup Tables job.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0ae39b5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0ae39b5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0ae39b5b
Branch: refs/heads/sync
Commit: 0ae39b5bb1cb05d3660ffb94c125cd8e85b91902
Parents: 7d5fb85
Author: yiming.xu <10...@qq.com>
Authored: Tue Jan 16 14:04:10 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/source/hive/HiveMRInput.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0ae39b5b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index e4564d0..6f42961 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -210,6 +210,9 @@ public class HiveMRInput implements IMRInput {
if (lookupViewsTables.size() == 0) {
return null;
}
+ // Create work dir to avoid hive create it,
+ // the difference is that the owners are different.
+ checkAndCreateWorkDir(jobWorkingDir);
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.overwriteHiveProps(kylinConfig.getHiveConfigOverride());
@@ -239,6 +242,19 @@ public class HiveMRInput implements IMRInput {
return step;
}
+ private void checkAndCreateWorkDir(String jobWorkingDir) {
+ try {
+ Path path = new Path(jobWorkingDir);
+ FileSystem fileSystem = HadoopUtil.getFileSystem(path);
+ if (!fileSystem.exists(path)) {
+ logger.info("Create jobWorkDir : " + jobWorkingDir);
+ fileSystem.mkdirs(path);
+ }
+ } catch (IOException e) {
+ logger.error("Could not create lookUp table dir : " + jobWorkingDir);
+ }
+ }
+
private AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
String cubeName) {
//from hive to hive
[38/50] [abbrv] kylin git commit: KYLIN-3209: Make optimize job
partial statistics path consistent with batch building ones
Posted by li...@apache.org.
KYLIN-3209: Make optimize job partial statistics path consistent with batch building ones
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f63fa61
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f63fa61
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f63fa61
Branch: refs/heads/sync
Commit: 0f63fa617ffc9604e61ba024f8dcf8acb30a54a2
Parents: 5eedd13
Author: Ma Gang <mg...@163.com>
Authored: Tue Jan 30 11:17:33 2018 +0800
Committer: Zhong <nj...@apache.org>
Committed: Tue Jan 30 11:24:41 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java | 4 +++-
.../apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f63fa61/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
index b1e59a7..f50a4be 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsWriter.java
@@ -50,10 +50,12 @@ public class CubeStatsWriter {
mapperOverlapRatio);
}
+ //Be care of that the file name for partial cuboid statistics should start with BatchConstants.CFG_OUTPUT_STATISTICS,
+ //Then for later statistics merging, only files starting with BatchConstants.CFG_OUTPUT_STATISTICS will be used
public static void writePartialCuboidStatistics(Configuration conf, Path outputPath, //
Map<Long, HLLCounter> cuboidHLLMap, int samplingPercentage, int mapperNumber, double mapperOverlapRatio,
int shard) throws IOException {
- Path seqFilePath = new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME + "_" + shard);
+ Path seqFilePath = new Path(outputPath, BatchConstants.CFG_OUTPUT_STATISTICS + "_" + shard);
writeCuboidStatisticsInner(conf, seqFilePath, cuboidHLLMap, samplingPercentage, mapperNumber,
mapperOverlapRatio);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f63fa61/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
index 3f12b0d..434892c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsWithOldStep.java
@@ -86,7 +86,7 @@ public class MergeStatisticsWithOldStep extends AbstractExecutable {
}
Path[] statisticsFiles = HadoopUtil.getFilteredPath(hdfs, statisticsDirPath,
- BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
+ BatchConstants.CFG_OUTPUT_STATISTICS);
if (statisticsFiles == null) {
throw new IOException("fail to find the statistics file in base dir: " + statisticsDirPath);
}
[05/50] [abbrv] kylin git commit: minor refactor
Posted by li...@apache.org.
minor refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0988c846
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0988c846
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0988c846
Branch: refs/heads/sync
Commit: 0988c846823ca65a46eb6ab9b3dcee443da57872
Parents: 0ae39b5
Author: Hongbin Ma <ma...@kyligence.io>
Authored: Thu Jan 11 16:47:37 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/request/ModelRequest.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0988c846/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
index 5f07997..2c5b3bf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/ModelRequest.java
@@ -26,6 +26,8 @@ public class ModelRequest {
private boolean successful;
private String message;
private String project;
+
+ private String ccInCheck;
private boolean isSeekingExprAdvice;
public String getUuid() {
@@ -112,4 +114,12 @@ public class ModelRequest {
public void setIsSeekingExprAdvice(boolean seekingExprAdvice) {
isSeekingExprAdvice = seekingExprAdvice;
}
+
+ public String getCcInCheck() {
+ return ccInCheck;
+ }
+
+ public void setCcInCheck(String ccInCheck) {
+ this.ccInCheck = ccInCheck;
+ }
}
[09/50] [abbrv] kylin git commit: KYLIN-3172 No such file or
directory error with CreateLookupHiveViewMaterializationStep
Posted by li...@apache.org.
KYLIN-3172 No such file or directory error with CreateLookupHiveViewMaterializationStep
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/35979ff0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/35979ff0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/35979ff0
Branch: refs/heads/sync
Commit: 35979ff0875d06cb4b1cc4b151efe6af37d1318d
Parents: 195fdef
Author: yiming.xu <10...@qq.com>
Authored: Wed Jan 17 18:37:08 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../kylin/common/util/HiveCmdBuilder.java | 45 ++++++---------
.../kylin/common/util/HiveCmdBuilderTest.java | 33 +++++++----
.../apache/kylin/source/hive/HiveMRInput.java | 36 ++++++------
.../kylin/source/hive/HiveMRInputTest.java | 59 ++++++++++++++++++++
4 files changed, 115 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
index add53db..c33a7fb 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java
@@ -18,11 +18,7 @@
package org.apache.kylin.common.util;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
@@ -30,8 +26,6 @@ import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.Logger;
@@ -42,9 +36,10 @@ import org.w3c.dom.NodeList;
import com.google.common.collect.Lists;
public class HiveCmdBuilder {
- private static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class);
+ public static final Logger logger = LoggerFactory.getLogger(HiveCmdBuilder.class);
public static final String HIVE_CONF_FILENAME = "kylin_hive_conf";
+ static final String CREATE_HQL_TMP_FILE_TEMPLATE = "cat >%s<<EOL\n%sEOL";
public enum HiveClientMode {
CLI, BEELINE
@@ -68,10 +63,11 @@ public class HiveCmdBuilder {
beelineShell = kylinConfig.getSparkSqlBeelineShell();
beelineParams = kylinConfig.getSparkSqlBeelineParams();
if (StringUtils.isBlank(beelineShell)) {
- throw new IllegalStateException("Missing config 'kylin.source.hive.sparksql-beeline-shell', please check kylin.properties");
+ throw new IllegalStateException(
+ "Missing config 'kylin.source.hive.sparksql-beeline-shell', please check kylin.properties");
}
}
-
+
StringBuffer buf = new StringBuffer();
switch (clientMode) {
@@ -84,37 +80,28 @@ public class HiveCmdBuilder {
buf.append(parseProps());
break;
case BEELINE:
- BufferedWriter bw = null;
- File tmpHql = null;
+ String tmpHqlPath = null;
+ StringBuilder hql = new StringBuilder();
try {
- tmpHql = File.createTempFile("beeline_", ".hql");
- bw = new BufferedWriter(new FileWriter(tmpHql));
+ tmpHqlPath = "/tmp/" + System.currentTimeMillis() + ".hql";
for (String statement : statements) {
- bw.write(statement);
- bw.newLine();
+ hql.append(statement);
+ hql.append("\n");
}
+ String createFileCmd = String.format(CREATE_HQL_TMP_FILE_TEMPLATE, tmpHqlPath, hql);
+ buf.append(createFileCmd);
+ buf.append("\n");
buf.append(beelineShell);
buf.append(" ");
buf.append(beelineParams);
buf.append(parseProps());
buf.append(" -f ");
- buf.append(tmpHql.getAbsolutePath());
+ buf.append(tmpHqlPath);
buf.append(";ret_code=$?;rm -f ");
- buf.append(tmpHql.getAbsolutePath());
+ buf.append(tmpHqlPath);
buf.append(";exit $ret_code");
-
- } catch (IOException e) {
- throw new RuntimeException(e);
} finally {
- IOUtils.closeQuietly(bw);
-
- if (tmpHql != null && logger.isDebugEnabled()) {
- String hql = null;
- try {
- hql = FileUtils.readFileToString(tmpHql, Charset.defaultCharset());
- } catch (IOException e) {
- // ignore
- }
+ if (tmpHqlPath != null && logger.isDebugEnabled()) {
logger.debug("The SQL to execute in beeline: \n" + hql);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
index 4262722..ecc8961 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/HiveCmdBuilderTest.java
@@ -45,7 +45,7 @@ public class HiveCmdBuilderTest {
System.clearProperty("kylin.source.hive.client");
System.clearProperty("kylin.source.hive.beeline-shell");
System.clearProperty("kylin.source.hive.beeline-params");
-
+
System.clearProperty("kylin.source.hive.enable-sparksql-for-table-ops");
System.clearProperty("kylin.source.hive.sparksql-beeline-shell");
System.clearProperty("kylin.source.hive.sparksql-beeline-params");
@@ -65,12 +65,15 @@ public class HiveCmdBuilderTest {
hiveCmdBuilder.addStatement("SHOW\n TABLES;");
hiveCmdBuilder.setHiveConfProps(hiveProps);
hiveCmdBuilder.overwriteHiveProps(hivePropsOverwrite);
- assertEquals("hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez", hiveCmdBuilder.build());
+ assertEquals(
+ "hive -e \"USE default;\nDROP TABLE test;\nSHOW\n TABLES;\n\" --hiveconf hive.execution.engine=tez",
+ hiveCmdBuilder.build());
}
@Test
public void testBeeline() throws IOException {
- String lineSeparator = java.security.AccessController.doPrivileged(new sun.security.action.GetPropertyAction("line.separator"));
+ String lineSeparator = java.security.AccessController
+ .doPrivileged(new sun.security.action.GetPropertyAction("line.separator"));
System.setProperty("kylin.source.hive.client", "beeline");
System.setProperty("kylin.source.hive.beeline-shell", "/spark-client/bin/beeline");
System.setProperty("kylin.source.hive.beeline-params", "-u jdbc_url");
@@ -78,17 +81,19 @@ public class HiveCmdBuilderTest {
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement("USE default;");
hiveCmdBuilder.addStatement("DROP TABLE test;");
- hiveCmdBuilder.addStatement("SHOW\n TABLES;");
+ hiveCmdBuilder.addStatement("SHOW TABLES;");
String cmd = hiveCmdBuilder.build();
- assertTrue(cmd.startsWith("/spark-client/bin/beeline -u jdbc_url"));
-
String hqlFile = cmd.substring(cmd.lastIndexOf("-f ") + 3).trim();
hqlFile = hqlFile.substring(0, hqlFile.length() - ";exit $ret_code".length());
-
+ String createFileCmd = cmd.substring(0, cmd.indexOf("EOL\n", cmd.indexOf("EOL\n") + 1) + 3);
+ CliCommandExecutor cliCommandExecutor = new CliCommandExecutor();
+ Pair<Integer, String> execute = cliCommandExecutor.execute(createFileCmd);
String hqlStatement = FileUtils.readFileToString(new File(hqlFile), Charset.defaultCharset());
- assertEquals("USE default;" + lineSeparator + "DROP TABLE test;" + lineSeparator + "SHOW\n TABLES;" + lineSeparator, hqlStatement);
-
+ assertEquals(
+ "USE default;" + lineSeparator + "DROP TABLE test;" + lineSeparator + "SHOW TABLES;" + lineSeparator,
+ hqlStatement);
+ assertBeelineCmd(cmd);
FileUtils.forceDelete(new File(hqlFile));
}
@@ -101,9 +106,13 @@ public class HiveCmdBuilderTest {
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement("USE default;");
hiveCmdBuilder.addStatement("DROP TABLE test;");
- hiveCmdBuilder.addStatement("SHOW\n TABLES;");
-
+ hiveCmdBuilder.addStatement("SHOW TABLES;");
String cmd = hiveCmdBuilder.build();
- assertTrue(cmd.startsWith("/spark-client/bin/beeline -u jdbc_url"));
+ assertBeelineCmd(cmd);
+ }
+
+ private void assertBeelineCmd(String cmd) {
+ String beelineCmd = cmd.substring(cmd.indexOf("EOL\n", cmd.indexOf("EOL\n") + 1) + 4);
+ assertTrue(beelineCmd.startsWith("/spark-client/bin/beeline -u jdbc_url"));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 6f42961..31a99cd 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -177,7 +177,25 @@ public class HiveMRInput implements IMRInput {
}
protected String getJobWorkingDir(DefaultChainedExecutable jobFlow) {
- return JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
+
+ String jobWorkingDir = JobBuilderSupport.getJobWorkingDir(hdfsWorkingDir, jobFlow.getId());
+ // Create work dir to avoid hive create it,
+ // the difference is that the owners are different.
+ checkAndCreateWorkDir(jobWorkingDir);
+ return jobWorkingDir;
+ }
+
+ private void checkAndCreateWorkDir(String jobWorkingDir) {
+ try {
+ Path path = new Path(jobWorkingDir);
+ FileSystem fileSystem = HadoopUtil.getFileSystem(path);
+ if (!fileSystem.exists(path)) {
+ logger.info("Create jobWorkDir : " + jobWorkingDir);
+ fileSystem.mkdirs(path);
+ }
+ } catch (IOException e) {
+ logger.error("Could not create lookUp table dir : " + jobWorkingDir);
+ }
}
private AbstractExecutable createRedistributeFlatHiveTableStep(String hiveInitStatements, String cubeName) {
@@ -210,9 +228,6 @@ public class HiveMRInput implements IMRInput {
if (lookupViewsTables.size() == 0) {
return null;
}
- // Create work dir to avoid hive create it,
- // the difference is that the owners are different.
- checkAndCreateWorkDir(jobWorkingDir);
HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.overwriteHiveProps(kylinConfig.getHiveConfigOverride());
@@ -242,19 +257,6 @@ public class HiveMRInput implements IMRInput {
return step;
}
- private void checkAndCreateWorkDir(String jobWorkingDir) {
- try {
- Path path = new Path(jobWorkingDir);
- FileSystem fileSystem = HadoopUtil.getFileSystem(path);
- if (!fileSystem.exists(path)) {
- logger.info("Create jobWorkDir : " + jobWorkingDir);
- fileSystem.mkdirs(path);
- }
- } catch (IOException e) {
- logger.error("Could not create lookUp table dir : " + jobWorkingDir);
- }
- }
-
private AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
String cubeName) {
//from hive to hive
http://git-wip-us.apache.org/repos/asf/kylin/blob/35979ff0/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
new file mode 100644
index 0000000..6d0737d
--- /dev/null
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/HiveMRInputTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class HiveMRInputTest {
+
+ @Test
+ public void TestGetJobWorkingDir() throws IOException {
+ FileSystem fileSystem = FileSystem.get(new Configuration());
+ Path jobWorkDirPath = null;
+ try {
+ KylinConfig kylinConfig = mock(KylinConfig.class);
+ KylinConfig.setKylinConfigThreadLocal(kylinConfig);
+ when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/");
+
+ DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class);
+ defaultChainedExecutable.setId(UUID.randomUUID().toString());
+
+ HiveMRInput.BatchCubingInputSide batchCubingInputSide = new HiveMRInput.BatchCubingInputSide(null);
+ String jobWorkingDir = batchCubingInputSide.getJobWorkingDir(defaultChainedExecutable);
+ jobWorkDirPath = new Path(jobWorkingDir);
+ Assert.assertTrue(fileSystem.exists(jobWorkDirPath));
+ } finally {
+ if (jobWorkDirPath != null)
+ fileSystem.deleteOnExit(jobWorkDirPath);
+ }
+ }
+
+}
\ No newline at end of file
[06/50] [abbrv] kylin git commit: minor, add array type test
Posted by li...@apache.org.
minor, add array type test
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ef3d490
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ef3d490
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ef3d490
Branch: refs/heads/sync
Commit: 2ef3d4906937ad1b00bc344f8cfcbb8900ce0998
Parents: a4acb4d
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jan 4 16:52:23 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/query/schema/OLAPTable.java | 2 +-
.../kylin/query/schema/OLAPTableTest.java | 78 ++++++++++++++++++++
2 files changed, 79 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2ef3d490/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index ca8807e..b5313ed 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -88,7 +88,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
SQLTYPE_MAPPING.put("timestamp", SqlTypeName.TIMESTAMP);
SQLTYPE_MAPPING.put("any", SqlTypeName.ANY);
- REGEX_SQLTYPE_MAPPING.put("array\\<.*\\>", SqlTypeName.ARRAY);
+ REGEX_SQLTYPE_MAPPING.put("array\\<.+\\>", SqlTypeName.ARRAY);
}
private final boolean exposeMore;
http://git-wip-us.apache.org/repos/asf/kylin/blob/2ef3d490/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java b/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
new file mode 100644
index 0000000..00440de
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/schema/OLAPTableTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.schema;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.sql.type.ArraySqlType;
+import org.apache.calcite.sql.type.BasicSqlType;
+import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class OLAPTableTest extends LocalFileMetadataTestCase {
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testCreateSqlType() {
+ RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
+ DataType kylinDataType = DataType.getType("array<string>");
+
+ RelDataType relDataType = OLAPTable.createSqlType(typeFactory, kylinDataType, true);
+ Assert.assertTrue(relDataType instanceof ArraySqlType);
+ Assert.assertEquals(SqlTypeName.ARRAY, relDataType.getSqlTypeName());
+ Assert.assertTrue(relDataType.getComponentType() instanceof BasicSqlType);
+ Assert.assertEquals(SqlTypeName.VARCHAR, relDataType.getComponentType().getSqlTypeName());
+ Assert.assertTrue(relDataType.isNullable());
+
+ kylinDataType = DataType.getType("array<>");
+ boolean catchedEx = false;
+ try {
+ OLAPTable.createSqlType(typeFactory, kylinDataType, true);
+ } catch (IllegalArgumentException ex) {
+ Assert.assertEquals("Unrecognized data type array<>", ex.getMessage());
+ catchedEx = true;
+ }
+ Assert.assertTrue(catchedEx);
+
+ kylinDataType = DataType.getType("array<AAA>");
+ catchedEx = false;
+ try {
+ OLAPTable.createSqlType(typeFactory, kylinDataType, true);
+ } catch (IllegalArgumentException ex) {
+ Assert.assertTrue(ex.getMessage().startsWith("bad data type -- aaa"));
+ catchedEx = true;
+ }
+ Assert.assertTrue(catchedEx);
+ }
+}
[47/50] [abbrv] kylin git commit: KYLIN-3194 use BrokenExecutable to
tolerate missing job classes
Posted by li...@apache.org.
KYLIN-3194 use BrokenExecutable to tolerate missing job classes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f797c840
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f797c840
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f797c840
Branch: refs/heads/sync
Commit: f797c84044f52f9c351ad680bc3a8a3482781121
Parents: c771833
Author: Li Yang <li...@apache.org>
Authored: Wed Jan 24 15:10:07 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Feb 4 08:44:22 2018 +0800
----------------------------------------------------------------------
.../kylin/job/execution/AbstractExecutable.java | 2 +-
.../kylin/job/execution/BrokenExecutable.java | 47 +++++++
.../kylin/job/execution/ExecutableManager.java | 124 ++++---------------
.../apache/kylin/job/ExecutableManagerTest.java | 16 +--
.../d9a2b721-9916-4607-8047-148ceb2473b1 | 14 +++
.../apache/kylin/rest/service/JobService.java | 9 +-
6 files changed, 102 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f797c840/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index 1a84871..91283f0 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -259,7 +259,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
}
@Override
- public final ExecutableState getStatus() {
+ public ExecutableState getStatus() {
ExecutableManager manager = getManager();
return manager.getOutput(this.getId()).getState();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f797c840/core-job/src/main/java/org/apache/kylin/job/execution/BrokenExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/BrokenExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/BrokenExecutable.java
new file mode 100644
index 0000000..7bb4fe2
--- /dev/null
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/BrokenExecutable.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.execution;
+
+import org.apache.kylin.job.exception.ExecuteException;
+
+/**
+ * A special Executable used to indicate any executable whose metadata is broken.
+ */
+public class BrokenExecutable extends AbstractExecutable {
+
+ public BrokenExecutable() {
+ super();
+ }
+
+ @Override
+ public String getName() {
+ return "[BROKEN] " + super.getName();
+ }
+
+ @Override
+ public ExecutableState getStatus() {
+ return ExecutableState.DISCARDED;
+ }
+
+ @Override
+ protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
+ throw new UnsupportedOperationException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f797c840/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 83582d2..0069c03 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -23,7 +23,6 @@ import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_ID;
import static org.apache.kylin.job.constant.ExecutableConstants.YARN_APP_URL;
import java.io.IOException;
-import java.lang.reflect.Constructor;
import java.util.HashMap;
import java.util.IllegalFormatException;
import java.util.List;
@@ -239,45 +238,6 @@ public class ExecutableManager {
}
}
- /**
- * Since ExecutableManager will instantiate all AbstractExecutable class by Class.forName(), but for each version release,
- * new classes are introduced, old classes are deprecated, renamed or removed. The Class.forName() will throw out
- * ClassNotFoundException. This API is used to retrieve the Executable Object list, not for calling the object method,
- * so we could just instance the parent common class instead of the concrete class. It will tolerate the class missing issue.
- *
- * @param timeStartInMillis
- * @param timeEndInMillis
- * @param expectedClass
- * @return
- */
- public List<AbstractExecutable> getAllAbstractExecutables(long timeStartInMillis, long timeEndInMillis,
- Class<? extends AbstractExecutable> expectedClass) {
- try {
- List<AbstractExecutable> ret = Lists.newArrayList();
- for (ExecutablePO po : executableDao.getJobs(timeStartInMillis, timeEndInMillis)) {
- try {
- AbstractExecutable ae = parseToAbstract(po, expectedClass);
- ret.add(ae);
- } catch (IllegalArgumentException e) {
- logger.error("error parsing one executabePO: ", e);
- }
- }
- return ret;
- } catch (PersistentException e) {
- logger.error("error get All Jobs", e);
- throw new RuntimeException(e);
- }
- }
-
- public AbstractExecutable getAbstractExecutable(String uuid, Class<? extends AbstractExecutable> expectedClass) {
- try {
- return parseToAbstract(executableDao.getJob(uuid), expectedClass);
- } catch (PersistentException e) {
- logger.error("fail to get job:" + uuid, e);
- throw new RuntimeException(e);
- }
- }
-
public List<String> getAllJobIds() {
try {
return executableDao.getJobIds();
@@ -536,70 +496,40 @@ public class ExecutableManager {
return null;
}
String type = executablePO.getType();
- try {
- Class<? extends AbstractExecutable> clazz = ClassUtil.forName(type, AbstractExecutable.class);
- Constructor<? extends AbstractExecutable> constructor = clazz.getConstructor();
- AbstractExecutable result = constructor.newInstance();
- result.initConfig(config);
- result.setId(executablePO.getUuid());
- result.setName(executablePO.getName());
- result.setParams(executablePO.getParams());
- List<ExecutablePO> tasks = executablePO.getTasks();
- if (tasks != null && !tasks.isEmpty()) {
- Preconditions.checkArgument(result instanceof ChainedExecutable);
- for (ExecutablePO subTask : tasks) {
- ((ChainedExecutable) result).addTask(parseTo(subTask));
- }
+ AbstractExecutable result = newExecutable(type);
+ result.initConfig(config);
+ result.setId(executablePO.getUuid());
+ result.setName(executablePO.getName());
+ result.setParams(executablePO.getParams());
+ List<ExecutablePO> tasks = executablePO.getTasks();
+ if (tasks != null && !tasks.isEmpty()) {
+ Preconditions.checkArgument(result instanceof ChainedExecutable);
+ for (ExecutablePO subTask : tasks) {
+ ((ChainedExecutable) result).addTask(parseTo(subTask));
}
- List<ExecutablePO> tasksForCheck = executablePO.getTasksForCheck();
- if (tasksForCheck != null && !tasksForCheck.isEmpty()) {
- Preconditions.checkArgument(result instanceof CheckpointExecutable);
- for (ExecutablePO subTaskForCheck : tasksForCheck) {
- ((CheckpointExecutable) result).addTaskForCheck(parseTo(subTaskForCheck));
- }
+ }
+ List<ExecutablePO> tasksForCheck = executablePO.getTasksForCheck();
+ if (tasksForCheck != null && !tasksForCheck.isEmpty()) {
+ Preconditions.checkArgument(result instanceof CheckpointExecutable);
+ for (ExecutablePO subTaskForCheck : tasksForCheck) {
+ ((CheckpointExecutable) result).addTaskForCheck(parseTo(subTaskForCheck));
}
- return result;
- } catch (ReflectiveOperationException e) {
- throw new IllegalStateException("cannot parse this job:" + executablePO.getId(), e);
}
+ return result;
}
- private AbstractExecutable parseToAbstract(ExecutablePO executablePO,
- Class<? extends AbstractExecutable> expectedClass) {
- if (executablePO == null) {
- logger.warn("executablePO is null");
- return null;
+ private AbstractExecutable newExecutable(String type) {
+ Class<? extends AbstractExecutable> clazz;
+ try {
+ clazz = ClassUtil.forName(type, AbstractExecutable.class);
+ } catch (ClassNotFoundException ex) {
+ clazz = BrokenExecutable.class;
+ logger.error("Unknown executable type '" + type + "', using BrokenExecutable");
}
- String type = executablePO.getType();
try {
- Class<? extends AbstractExecutable> clazz = null;
- try {
- clazz = ClassUtil.forName(type, AbstractExecutable.class);
- } catch (ClassNotFoundException e) {
- clazz = ClassUtil.forName(expectedClass.getName(), AbstractExecutable.class);
- }
- Constructor<? extends AbstractExecutable> constructor = clazz.getConstructor();
- AbstractExecutable result = constructor.newInstance();
- result.initConfig(config);
- result.setId(executablePO.getUuid());
- result.setName(executablePO.getName());
- result.setParams(executablePO.getParams());
- List<ExecutablePO> tasks = executablePO.getTasks();
- if (tasks != null && !tasks.isEmpty()) {
- Preconditions.checkArgument(result instanceof ChainedExecutable);
- for (ExecutablePO subTask : tasks) {
- AbstractExecutable parseToTask = null;
- try {
- parseToTask = parseTo(subTask);
- } catch (IllegalStateException e) {
- parseToTask = parseToAbstract(subTask, DefaultChainedExecutable.class);
- }
- ((ChainedExecutable) result).addTask(parseToTask);
- }
- }
- return result;
- } catch (ReflectiveOperationException e) {
- throw new IllegalStateException("cannot parse this job:" + executablePO.getId(), e);
+ return clazz.getConstructor().newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to instantiate " + clazz, e);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f797c840/core-job/src/test/java/org/apache/kylin/job/ExecutableManagerTest.java
----------------------------------------------------------------------
diff --git a/core-job/src/test/java/org/apache/kylin/job/ExecutableManagerTest.java b/core-job/src/test/java/org/apache/kylin/job/ExecutableManagerTest.java
index faea9a4..73f0410 100644
--- a/core-job/src/test/java/org/apache/kylin/job/ExecutableManagerTest.java
+++ b/core-job/src/test/java/org/apache/kylin/job/ExecutableManagerTest.java
@@ -47,12 +47,6 @@ public class ExecutableManagerTest extends LocalFileMetadataTestCase {
public void setup() throws Exception {
createTestMetadata();
service = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
-
- for (String jobId : service.getAllJobIds()) {
- System.out.println("deleting " + jobId);
- service.deleteJob(jobId);
- }
-
}
@After
@@ -63,13 +57,21 @@ public class ExecutableManagerTest extends LocalFileMetadataTestCase {
@Test
public void test() throws Exception {
assertNotNull(service);
+
+ // all existing are broken jobs
+ List<AbstractExecutable> existing = service.getAllExecutables();
+ for (AbstractExecutable exec : existing) {
+ assertEquals("BrokenExecutable", exec.getClass().getSimpleName());
+ assertEquals(ExecutableState.DISCARDED, exec.getStatus());
+ }
+
BaseTestExecutable executable = new SucceedTestExecutable();
executable.setParam("test1", "test1");
executable.setParam("test2", "test2");
executable.setParam("test3", "test3");
service.addJob(executable);
List<AbstractExecutable> result = service.getAllExecutables();
- assertEquals(1, result.size());
+ assertEquals(existing.size() + 1, result.size());
AbstractExecutable another = service.getJob(executable.getId());
assertJobEqual(executable, another);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f797c840/examples/test_case_data/localmeta/execute/d9a2b721-9916-4607-8047-148ceb2473b1
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/execute/d9a2b721-9916-4607-8047-148ceb2473b1 b/examples/test_case_data/localmeta/execute/d9a2b721-9916-4607-8047-148ceb2473b1
new file mode 100644
index 0000000..fb8ff30
--- /dev/null
+++ b/examples/test_case_data/localmeta/execute/d9a2b721-9916-4607-8047-148ceb2473b1
@@ -0,0 +1,14 @@
+{
+ "uuid" : "d9a2b721-9916-4607-8047-148ceb2473b1",
+ "last_modified" : 1516778161249,
+ "version" : "2.3.0",
+ "name" : null,
+ "tasks" : null,
+ "tasks_check" : null,
+ "type" : "org.apache.kylin.job.BadClassName",
+ "params" : {
+ "test2" : "test2",
+ "test3" : "test3",
+ "test1" : "test1"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f797c840/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index e447031..5f80b84 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -725,9 +725,9 @@ public class JobService extends BasicService implements InitializingBean {
public List<CubingJob> innerSearchCubingJobs(final String cubeName, final String jobName,
final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis,
final Map<String, Output> allOutputs, final boolean nameExactMatch, final String projectName) {
- List<CubingJob> results = Lists.newArrayList(FluentIterable.from(
- getExecutableManager().getAllAbstractExecutables(timeStartInMillis, timeEndInMillis, CubingJob.class))
- .filter(new Predicate<AbstractExecutable>() {
+ List<CubingJob> results = Lists.newArrayList(
+ FluentIterable.from(getExecutableManager().getAllExecutables(timeStartInMillis, timeEndInMillis))
+ .filter(new Predicate<AbstractExecutable>() {
@Override
public boolean apply(AbstractExecutable executable) {
if (executable instanceof CubingJob) {
@@ -824,8 +824,7 @@ public class JobService extends BasicService implements InitializingBean {
List<CheckpointExecutable> results = Lists
.newArrayList(
FluentIterable
- .from(getExecutableManager().getAllAbstractExecutables(timeStartInMillis,
- timeEndInMillis, CheckpointExecutable.class))
+ .from(getExecutableManager().getAllExecutables(timeStartInMillis, timeEndInMillis))
.filter(new Predicate<AbstractExecutable>() {
@Override
public boolean apply(AbstractExecutable executable) {
[11/50] [abbrv] kylin git commit: KYLIN-2876,
refine singleton in ExternalAclProvider.
Posted by li...@apache.org.
KYLIN-2876, refine singleton in ExternalAclProvider.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a31bce69
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a31bce69
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a31bce69
Branch: refs/heads/sync
Commit: a31bce69dfef5ee8070e8b642cdcfbcfde404a60
Parents: 55365b6
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 8 19:35:53 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../rest/security/ExternalAclProvider.java | 28 +++++++++-----------
1 file changed, 12 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a31bce69/server-base/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java b/server-base/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java
index 9e977d0..ef21f00 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/ExternalAclProvider.java
@@ -19,6 +19,7 @@ package org.apache.kylin.rest.security;
import java.util.List;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Pair;
@@ -28,26 +29,21 @@ import org.springframework.security.acls.model.Permission;
*/
abstract public class ExternalAclProvider {
- private static boolean inited = false;
- private static ExternalAclProvider singleton = null;
+ private volatile static ExternalAclProvider singleton = null;
public static ExternalAclProvider getInstance() {
- if (inited)
- return singleton;
-
- synchronized (ExternalAclProvider.class) {
- if (inited)
- return singleton;
-
- String cls = KylinConfig.getInstanceFromEnv().getExternalAclProvider();
- if (cls != null && cls.length() > 0) {
- singleton = (ExternalAclProvider) ClassUtil.newInstance(cls);
- singleton.init();
+ if (singleton == null) {
+ synchronized (ExternalAclProvider.class) {
+ if (singleton == null) {
+ String cls = KylinConfig.getInstanceFromEnv().getExternalAclProvider();
+ if (!StringUtils.isBlank(cls)) {
+ singleton = (ExternalAclProvider) ClassUtil.newInstance(cls);
+ singleton.init();
+ }
+ }
}
-
- inited = true;
- return singleton;
}
+ return singleton;
}
// ============================================================================
[28/50] [abbrv] kylin git commit: KYLIN-3125 bug fix
Posted by li...@apache.org.
KYLIN-3125 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0fce61c8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0fce61c8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0fce61c8
Branch: refs/heads/sync
Commit: 0fce61c805f1822de6a2cc540bd96dfe46f23eb9
Parents: 48bccac
Author: Li Yang <li...@apache.org>
Authored: Thu Jan 11 13:21:14 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
core-common/src/main/resources/kylin-defaults.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0fce61c8/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index bbb5f55..a79728a 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -83,7 +83,7 @@ kylin.server.external-acl-provider=
kylin.source.hive.client=cli
# Absolute path to beeline shell, can be set to spark beeline instead of the default hive beeline on PATH
-#kylin.source.hive.beeline-shell=beeline
+kylin.source.hive.beeline-shell=beeline
# Parameters for beeline client, only necessary if hive client is beeline
#kylin.source.hive.beeline-params=-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u jdbc:hive2://localhost:10000
[04/50] [abbrv] kylin git commit: minor, refine sql comment removing
Posted by li...@apache.org.
minor, refine sql comment removing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7d5fb855
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7d5fb855
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7d5fb855
Branch: refs/heads/sync
Commit: 7d5fb855064e2b81cd3b154cdeeafec4e64f63c9
Parents: 60caf61
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Jan 16 15:17:46 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/query/util/QueryUtil.java | 17 ++++-------------
.../org/apache/kylin/query/util/QueryUtilTest.java | 16 +++++++++++++++-
2 files changed, 19 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d5fb855/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 22658f1..da18bd7 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -142,23 +142,14 @@ public class QueryUtil {
public static String removeCommentInSql(String sql1) {
// match two patterns, one is "-- comment", the other is "/* comment */"
- final String[] commentPatterns = new String[] { "--[^\r\n]*", "/\\*[\\s\\S]*?\\*/" };
+ final String[] commentPatterns = new String[] { "--.*?[\r\n]", "/\\*.*?\\*/" };
for (int i = 0; i < commentPatterns.length; i++) {
- String commentPattern = commentPatterns[i];
- Pattern pattern = Pattern.compile(commentPattern);
- Matcher matcher = pattern.matcher(sql1);
-
- while (matcher.find()) {
- if (matcher.start() == 0) {
- sql1 = sql1.substring(matcher.end()).trim();
- } else {
- sql1 = (sql1.substring(0, matcher.start()) + sql1.substring(matcher.end())).trim();
- }
- matcher = pattern.matcher(sql1);
- }
+ sql1 = sql1.replaceAll(commentPatterns[i], "");
}
+ sql1 = sql1.trim();
+
return sql1;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7d5fb855/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
index e7a7ecb..e3cd819 100644
--- a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -149,9 +149,23 @@ public class QueryUtilTest extends LocalFileMetadataTestCase {
}
{
- String sqlWithComment = "/* comment \n select count(*) from kylin_sales; \n */ " + originSql;
+ String sqlWithComment = "/* comment1/comment2 */ " + originSql;
Assert.assertEquals(originSql, QueryUtil.removeCommentInSql(sqlWithComment));
}
+ {
+ String sqlWithComment = "/* comment1 * comment2 */ " + originSql;
+ Assert.assertEquals(originSql, QueryUtil.removeCommentInSql(sqlWithComment));
+ }
+
+ {
+ String sqlWithComment = "/* comment1 * comment2 */ /* comment3 / comment4 */ -- comment 5\n" + originSql;
+ Assert.assertEquals(originSql, QueryUtil.removeCommentInSql(sqlWithComment));
+ }
+
+ {
+ String sqlWithComment = "/* comment1 * comment2 */ -- comment 5\n" + originSql + "/* comment3 / comment4 */";
+ Assert.assertEquals(originSql, QueryUtil.removeCommentInSql(sqlWithComment));
+ }
}
}
[14/50] [abbrv] kylin git commit: Minor,
fix NotClassFound issue in org.apache.hadoop.hive.ql.Driver
Posted by li...@apache.org.
Minor, fix NotClassFound issue in org.apache.hadoop.hive.ql.Driver
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/26bee662
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/26bee662
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/26bee662
Branch: refs/heads/sync
Commit: 26bee6625e7a09373a9b2c39db57af0c1a5a8460
Parents: ebc53b6
Author: Yifan Zhang <ev...@gmail.com>
Authored: Mon Jan 1 16:33:26 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../kylin/source/hive/BeelineHiveClient.java | 5 +--
.../apache/kylin/source/hive/CLIHiveClient.java | 39 +++++++-------------
.../kylin/source/hive/HiveMetadataExplorer.java | 29 ++++++++++-----
.../apache/kylin/source/hive/IHiveClient.java | 6 +--
.../apache/kylin/source/jdbc/JdbcExplorer.java | 9 ++++-
5 files changed, 43 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/26bee662/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index ee693c5..314402c 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -28,7 +28,6 @@ import java.sql.Statement;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.kylin.common.util.DBUtils;
import com.google.common.base.Preconditions;
@@ -117,12 +116,12 @@ public class BeelineHiveClient implements IHiveClient {
}
@Override
- public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
+ public void executeHQL(String hql) throws IOException {
throw new UnsupportedOperationException();
}
@Override
- public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
+ public void executeHQL(String[] hqls) throws IOException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/26bee662/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index e8a93bd..bc9f17e 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -22,17 +22,15 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HiveCmdBuilder;
+import org.apache.kylin.common.util.Pair;
import com.google.common.collect.Lists;
@@ -43,7 +41,6 @@ import com.google.common.collect.Lists;
*/
public class CLIHiveClient implements IHiveClient {
protected HiveConf hiveConf = null;
- protected Driver driver = null;
protected HiveMetaStoreClient metaStoreClient = null;
public CLIHiveClient() {
@@ -52,22 +49,25 @@ public class CLIHiveClient implements IHiveClient {
/**
* only used by Deploy Util
+ * @throws IOException
*/
@Override
- public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
- CommandProcessorResponse response = getDriver().run(hql);
- int retCode = response.getResponseCode();
- if (retCode != 0) {
- String err = response.getErrorMessage();
- throw new IOException("Failed to execute hql [" + hql + "], error message is: " + err);
+ public void executeHQL(String hql) throws IOException {
+ final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ hiveCmdBuilder.addStatement(hql);
+ Pair<Integer, String> response = KylinConfig.getInstanceFromEnv().getCliCommandExecutor()
+ .execute(hiveCmdBuilder.toString());
+ if (response.getFirst() != 0) {
+ throw new IllegalArgumentException("Failed to execute hql [" + hql + "], error message is: " + response.getSecond());
}
+
}
/**
* only used by Deploy Util
*/
@Override
- public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
+ public void executeHQL(String[] hqls) throws IOException {
for (String sql : hqls)
executeHQL(sql);
}
@@ -159,17 +159,4 @@ public class CLIHiveClient implements IHiveClient {
}
return result;
}
-
- /**
- * Get the hive ql driver to execute ddl or dml
- * @return
- */
- private Driver getDriver() {
- if (driver == null) {
- driver = new Driver(hiveConf);
- SessionState.start(new CliSessionState(hiveConf));
- }
-
- return driver;
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/26bee662/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index d80c546..cb3eb02 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -32,11 +32,15 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.source.ISampleDataDeployer;
import org.apache.kylin.source.ISourceMetadataExplorer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDataDeployer {
+ private static final Logger logger = LoggerFactory.getLogger(HiveClientFactory.class);
+
IHiveClient hiveClient = HiveClientFactory.getHiveClient();
-
+
@Override
public List<String> listDatabases() throws Exception {
return hiveClient.getHiveDbNames();
@@ -90,7 +94,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
tableExtDesc.setUuid(UUID.randomUUID().toString());
tableExtDesc.setLastModified(0);
tableExtDesc.init(prj);
-
+
tableExtDesc.addDataSourceProp("location", hiveTableMeta.sdLocation);
tableExtDesc.addDataSourceProp("owner", hiveTableMeta.owner);
tableExtDesc.addDataSourceProp("last_access_time", String.valueOf(hiveTableMeta.lastAccessTime));
@@ -114,10 +118,10 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
hiveClient.executeHQL(generateCreateSchemaSql(database));
}
- private String generateCreateSchemaSql(String schemaName){
+ private String generateCreateSchemaSql(String schemaName) {
return String.format("CREATE DATABASE IF NOT EXISTS %s", schemaName);
}
-
+
@Override
public void createSampleTable(TableDesc table) throws Exception {
hiveClient.executeHQL(generateCreateTableSql(table));
@@ -146,7 +150,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
return new String[] { dropsql, dropsql2, ddl.toString() };
}
-
+
@Override
public void loadSampleData(String tableName, String tmpDataDir) throws Exception {
hiveClient.executeHQL(generateLoadDataSql(tableName, tmpDataDir));
@@ -155,12 +159,12 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
private String generateLoadDataSql(String tableName, String tableFileDir) {
return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName + ".csv' OVERWRITE INTO TABLE " + tableName;
}
-
+
@Override
public void createWrapperView(String origTableName, String viewName) throws Exception {
hiveClient.executeHQL(generateCreateViewSql(viewName, origTableName));
}
-
+
private String[] generateCreateViewSql(String viewName, String tableName) {
String dropView = "DROP VIEW IF EXISTS " + viewName;
@@ -170,7 +174,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
return new String[] { dropView, dropTable, createSql };
}
-
+
private static String getHiveDataType(String javaDataType) {
String hiveDataType = javaDataType.toLowerCase().startsWith("varchar") ? "string" : javaDataType;
hiveDataType = javaDataType.toLowerCase().startsWith("integer") ? "int" : hiveDataType;
@@ -192,16 +196,21 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
String evalViewSql = "CREATE VIEW " + tmpDatabase + "." + tmpView + " as " + query;
try {
- hiveClient.executeHQL(new String[] { dropViewSql, evalViewSql });
+ logger.debug("Removing duplicate view {}", tmpView);
+ hiveClient.executeHQL(dropViewSql);
+ logger.debug("Creating view {} for query: {}", tmpView, query);
+ hiveClient.executeHQL(evalViewSql);
+ logger.debug("Evaluating query columns' metadata");
HiveTableMeta hiveTableMeta = hiveClient.getHiveTableMeta(tmpDatabase, tmpView);
return extractColumnFromMeta(hiveTableMeta);
} catch (Exception e) {
throw new RuntimeException("Cannot evalutate metadata of query: " + query, e);
} finally {
try {
+ logger.debug("Cleaning up.");
hiveClient.executeHQL(dropViewSql);
} catch (Exception e) {
- throw new RuntimeException("Cannot temp view of query: " + query, e);
+ logger.warn("Cannot drop temp view of query: {}", query, e);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/26bee662/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
index ca5312d..ec2bf7f 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
@@ -21,13 +21,11 @@ package org.apache.kylin.source.hive;
import java.io.IOException;
import java.util.List;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-
public interface IHiveClient {
- void executeHQL(String hql) throws CommandNeedRetryException, IOException;
+ void executeHQL(String hql) throws IOException;
- void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException;
+ void executeHQL(String[] hqls) throws IOException;
HiveTableMeta getHiveTableMeta(String database, String tableName) throws Exception;
http://git-wip-us.apache.org/repos/asf/kylin/blob/26bee662/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index 81e42bb..d96a68d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -243,7 +243,11 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
String evalViewSql = "CREATE VIEW " + tmpView + " as " + query;
try {
- executeSQL(new String[] { dropViewSql, evalViewSql });
+ logger.debug("Removing duplicate view {}", tmpView);
+ executeSQL(dropViewSql);
+ logger.debug("Creating view {} for query: {}", tmpView, query);
+ executeSQL(evalViewSql);
+ logger.debug("Evaluating query columns' metadata");
Connection con = SqlUtil.getConnection(dbconf);
DatabaseMetaData dbmd = con.getMetaData();
ResultSet rs = dbmd.getColumns(null, tmpDatabase, tmpView, null);
@@ -255,9 +259,10 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
throw new RuntimeException("Cannot evalutate metadata of query: " + query, e);
} finally {
try {
+ logger.debug("Cleaning up.");
executeSQL(dropViewSql);
} catch (Exception e) {
- throw new RuntimeException("Cannot temp view of query: " + query, e);
+ logger.warn("Cannot drop temp view of query: {}", query, e);
}
}
}
[24/50] [abbrv] kylin git commit: KYLIN-3144,
adopt Collections.emptyList() for empty list values.
Posted by li...@apache.org.
KYLIN-3144, adopt Collections.emptyList() for empty list values.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87a0058d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87a0058d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87a0058d
Branch: refs/heads/sync
Commit: 87a0058d9188d4e1140470e0df260548ade097a1
Parents: 5bf5099
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 8 19:18:24 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/rest/service/HybridService.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/87a0058d/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
index 912923a..de20e7c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.service;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
@@ -119,7 +120,7 @@ public class HybridService extends BasicService {
if (StringUtils.isEmpty(projectName)) {
hybridsInProject = new ArrayList(getHybridManager().listHybridInstances());
} else if (project == null) {
- return new ArrayList<>();
+ return Collections.emptyList();
} else {
List<RealizationEntry> realizationEntries = project.getRealizationEntries(RealizationType.HYBRID);
if (realizationEntries != null) {
@@ -134,7 +135,7 @@ public class HybridService extends BasicService {
if (StringUtils.isEmpty(modelName)) {
return hybridsInProject;
} else if (model == null) {
- return new ArrayList<>();
+ return Collections.emptyList();
} else {
List<HybridInstance> hybridsInModel = new ArrayList<HybridInstance>();
for (HybridInstance hybridInstance : hybridsInProject) {
[41/50] [abbrv] kylin git commit: KYLIN-3204 minor refine on loggings
Posted by li...@apache.org.
KYLIN-3204 minor refine on loggings
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fee57302
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fee57302
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fee57302
Branch: refs/heads/sync
Commit: fee5730258a5082999bcf8d3141aa4cb179b693c
Parents: b07a056
Author: lidongsjtu <li...@apache.org>
Authored: Tue Jan 30 17:28:58 2018 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Jan 30 17:28:58 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/source/jdbc/JdbcExplorer.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fee57302/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index 2dd8f9a..d858cd4 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -135,7 +135,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
return String.format("IF NOT EXISTS (SELECT name FROM sys.schemas WHERE name = N'%s') EXEC('CREATE SCHEMA"
+ " [%s] AUTHORIZATION [dbo]')", schemaName, schemaName);
} else {
- logger.error(String.format("unsupported dialect %s.", dialect));
+ logger.error("unsupported dialect {}.", dialect);
return null;
}
}
@@ -156,7 +156,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
return String.format("BULK INSERT %s FROM '%s/%s.csv' WITH(FIELDTERMINATOR = ',')", tableName, tableFileDir,
tableName);
} else {
- logger.error(String.format("unsupported dialect %s.", dialect));
+ logger.error("unsupported dialect {}.", dialect);
return null;
}
}
@@ -167,7 +167,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
}
private String[] generateCreateTableSql(TableDesc tableDesc) {
- logger.info(String.format("gen create table sql:%s", tableDesc));
+ logger.info("Generate create table sql: {}", tableDesc);
String tableIdentity = String.format("%s.%s", tableDesc.getDatabase().toUpperCase(), tableDesc.getName())
.toUpperCase();
String dropsql = "DROP TABLE IF EXISTS " + tableIdentity;
@@ -207,7 +207,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
private void executeSQL(String sql) throws SQLException {
Connection con = SqlUtil.getConnection(dbconf);
- logger.info(String.format("Executing sql : %s"), sql);
+ logger.info("Executing sql : {}", sql);
try {
SqlUtil.execUpdateSQL(con, sql);
} finally {
@@ -218,7 +218,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
private void executeSQL(String[] sqls) throws SQLException {
try (Connection con = SqlUtil.getConnection(dbconf)) {
for (String sql : sqls) {
- logger.info(String.format("Executing sql : %s"), sql);
+ logger.info("Executing sql : {}", sql);
SqlUtil.execUpdateSQL(con, sql);
}
}
[16/50] [abbrv] kylin git commit: KYLIN-3157, refine test.
Posted by li...@apache.org.
KYLIN-3157, refine test.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/48bccac0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/48bccac0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/48bccac0
Branch: refs/heads/sync
Commit: 48bccac0fd1966c89111e7e640ac4641d2eedccf
Parents: c2b6915
Author: tttMelody <24...@qq.com>
Authored: Tue Jan 9 16:49:11 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../rest/service/KyilnQueryTimeoutTest.java | 104 ------------------
.../rest/service/KylinQueryTimeoutTest.java | 107 +++++++++++++++++++
2 files changed, 107 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/48bccac0/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java b/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
deleted file mode 100644
index 25ff75b..0000000
--- a/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.rest.service;
-import java.sql.SQLException;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.exceptions.KylinTimeoutException;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.metadata.tuple.TupleInfo;
-import org.apache.kylin.query.security.QueryACLTestUtil;
-import org.apache.kylin.rest.request.SQLRequest;
-import org.apache.kylin.storage.IStorage;
-import org.apache.kylin.storage.IStorageQuery;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.StorageFactory;
-import org.hamcrest.CoreMatchers;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-public class KyilnQueryTimeoutTest extends LocalFileMetadataTestCase {
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @Before
- public void setUp() {
- this.createTestMetadata();
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- config.setProperty("kylin.storage.provider.2", MockQueryTimeoutStorage.class.getName());
- config.setProperty("kylin.storage.default", "2");
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- StorageFactory.clearCache();
- }
-
- @Test
- public void testQueryTimeout() throws SQLException {
- thrown.expectCause(CoreMatchers.isA(KylinTimeoutException.class));
- thrown.expectMessage(CoreMatchers.containsString("Kylin query timeout"));
- StorageFactory.clearCache();
- BadQueryDetector detector = new BadQueryDetector(100, BadQueryDetector.getSystemAvailMB() * 2, 100, 1);
- detector.start();
- SQLRequest request = new SQLRequest();
- request.setProject("default");
- request.setSql("select count(*) from STREAMING_TABLE");
- detector.queryStart(Thread.currentThread(), request, "ADMIN");
- try {
- QueryACLTestUtil.mockQuery("default", "select * from STREAMING_TABLE");
- } finally{
- detector.queryEnd(Thread.currentThread(), "timeout");
- detector.interrupt();
- }
- }
-
- public static class MockQueryTimeoutStorage implements IStorage {
-
- @Override
- public IStorageQuery createQuery(IRealization realization) {
- return new MockQueryTimeoutQuery();
- }
-
- @Override
- public <I> I adaptToBuildEngine(Class<I> engineInterface) {
- return null;
- }
- }
-
- private static class MockQueryTimeoutQuery implements IStorageQuery {
- @Override
- public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
- try {
- Thread.sleep(5 * 1000);
- } catch (InterruptedException e) {
- throw new KylinTimeoutException("Kylin query timeout");
- }
- return null;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/48bccac0/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java b/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
new file mode 100644
index 0000000..6a61b2f
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.service;
+import java.sql.SQLException;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exceptions.KylinTimeoutException;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
+import org.apache.kylin.query.security.QueryACLTestUtil;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.storage.IStorage;
+import org.apache.kylin.storage.IStorageQuery;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.StorageFactory;
+import org.hamcrest.CoreMatchers;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class KylinQueryTimeoutTest extends LocalFileMetadataTestCase {
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Before
+ public void setUp() {
+ this.createTestMetadata();
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ config.setProperty("kylin.storage.provider.2", MockQueryTimeoutStorage.class.getName());
+ config.setProperty("kylin.storage.default", "2");
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ StorageFactory.clearCache();
+ }
+
+ @Test
+ public void testQueryTimeout() throws SQLException {
+ thrown.expectCause(CoreMatchers.isA(KylinTimeoutException.class));
+ thrown.expectMessage(CoreMatchers.containsString("Kylin query timeout"));
+ StorageFactory.clearCache();
+ BadQueryDetector detector = new BadQueryDetector(100, BadQueryDetector.getSystemAvailMB() * 2, 100, 1);
+ detector.start();
+ SQLRequest request = new SQLRequest();
+ request.setProject("default");
+ request.setSql("select count(*) from STREAMING_TABLE");
+ detector.queryStart(Thread.currentThread(), request, "ADMIN");
+ try {
+ QueryACLTestUtil.mockQuery("default", "select * from STREAMING_TABLE");
+ } finally{
+ detector.queryEnd(Thread.currentThread(), "timeout");
+ detector.interrupt();
+ }
+ // every place that thrown KylinTimeoutException should reset the interrupt.
+ Assert.assertEquals(false, Thread.currentThread().isInterrupted());
+ }
+
+ public static class MockQueryTimeoutStorage implements IStorage {
+
+ @Override
+ public IStorageQuery createQuery(IRealization realization) {
+ return new MockQueryTimeoutQuery();
+ }
+
+ @Override
+ public <I> I adaptToBuildEngine(Class<I> engineInterface) {
+ return null;
+ }
+ }
+
+ private static class MockQueryTimeoutQuery implements IStorageQuery {
+ @Override
+ public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ try {
+ Thread.sleep(5 * 1000);
+ } catch (InterruptedException e) {
+ throw new KylinTimeoutException("Kylin query timeout");
+ }
+ return null;
+ }
+ }
+}
\ No newline at end of file
[12/50] [abbrv] kylin git commit: KYLIN-3083 fix potential overflow
in CubeHBaseRPC#getCoprocessorTimeoutMillis
Posted by li...@apache.org.
KYLIN-3083 fix potential overflow in CubeHBaseRPC#getCoprocessorTimeoutMillis
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5bf5099e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5bf5099e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5bf5099e
Branch: refs/heads/sync
Commit: 5bf5099e15cf732f4b77773061743558fb4508ef
Parents: 67f7197
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 8 19:07:54 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5bf5099e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 1e2fbd6..2e82140 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -293,7 +293,7 @@ public abstract class CubeHBaseRPC implements IGTStorage {
if (BackdoorToggles.getQueryTimeout() != -1) {
coopTimeout = BackdoorToggles.getQueryTimeout();
} else {
- coopTimeout = cubeSeg.getConfig().getQueryCoprocessorTimeoutSeconds() * 1000;
+ coopTimeout = cubeSeg.getConfig().getQueryCoprocessorTimeoutSeconds() * 1000L;
}
int rpcTimeout;
[45/50] [abbrv] kylin git commit: KYLIN-3195 Fix Apache RAT checks
Posted by li...@apache.org.
KYLIN-3195 Fix Apache RAT checks
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/490891e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/490891e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/490891e3
Branch: refs/heads/sync
Commit: 490891e39ca3176f67058a3f7cd4b15fe3941b99
Parents: ff8bece
Author: Billy Liu <bi...@apache.org>
Authored: Wed Jan 31 17:04:00 2018 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Jan 31 17:23:27 2018 +0800
----------------------------------------------------------------------
.../calcite/rel/rules/FilterJoinRule.java | 32 +++++++----------
.../resources/mail_templates/JOB_DISCARD.ftl | 17 +++++++++
.../main/resources/mail_templates/JOB_ERROR.ftl | 17 +++++++++
.../resources/mail_templates/JOB_SUCCEED.ftl | 17 +++++++++
.../mail_templates/METADATA_PERSIST_FAIL.ftl | 17 +++++++++
external/htrace/pom.xml | 38 +++++++++-----------
external/pom.xml | 38 +++++++++-----------
pom.xml | 11 ++++++
8 files changed, 124 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/490891e3/atopcalcite/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java b/atopcalcite/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
index 4fdde17..f758b40 100644
--- a/atopcalcite/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
+++ b/atopcalcite/src/main/java/org/apache/calcite/rel/rules/FilterJoinRule.java
@@ -1,25 +1,19 @@
/*
- * Copyright (C) 2016 Kyligence Inc. All rights reserved.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://kyligence.io
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * This software is the confidential and proprietary information of
- * Kyligence Inc. ("Confidential Information"). You shall not disclose
- * such Confidential Information and shall use it only in accordance
- * with the terms of the license agreement you entered into with
- * Kyligence Inc.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.calcite.rel.rules;
http://git-wip-us.apache.org/repos/asf/kylin/blob/490891e3/core-job/src/main/resources/mail_templates/JOB_DISCARD.ftl
----------------------------------------------------------------------
diff --git a/core-job/src/main/resources/mail_templates/JOB_DISCARD.ftl b/core-job/src/main/resources/mail_templates/JOB_DISCARD.ftl
index 9712982..ab0de2e 100644
--- a/core-job/src/main/resources/mail_templates/JOB_DISCARD.ftl
+++ b/core-job/src/main/resources/mail_templates/JOB_DISCARD.ftl
@@ -1,3 +1,20 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
http://git-wip-us.apache.org/repos/asf/kylin/blob/490891e3/core-job/src/main/resources/mail_templates/JOB_ERROR.ftl
----------------------------------------------------------------------
diff --git a/core-job/src/main/resources/mail_templates/JOB_ERROR.ftl b/core-job/src/main/resources/mail_templates/JOB_ERROR.ftl
index 6012037..1d2e0a7 100644
--- a/core-job/src/main/resources/mail_templates/JOB_ERROR.ftl
+++ b/core-job/src/main/resources/mail_templates/JOB_ERROR.ftl
@@ -1,3 +1,20 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
http://git-wip-us.apache.org/repos/asf/kylin/blob/490891e3/core-job/src/main/resources/mail_templates/JOB_SUCCEED.ftl
----------------------------------------------------------------------
diff --git a/core-job/src/main/resources/mail_templates/JOB_SUCCEED.ftl b/core-job/src/main/resources/mail_templates/JOB_SUCCEED.ftl
index f1fb45c..fdce289 100644
--- a/core-job/src/main/resources/mail_templates/JOB_SUCCEED.ftl
+++ b/core-job/src/main/resources/mail_templates/JOB_SUCCEED.ftl
@@ -1,3 +1,20 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
http://git-wip-us.apache.org/repos/asf/kylin/blob/490891e3/core-job/src/main/resources/mail_templates/METADATA_PERSIST_FAIL.ftl
----------------------------------------------------------------------
diff --git a/core-job/src/main/resources/mail_templates/METADATA_PERSIST_FAIL.ftl b/core-job/src/main/resources/mail_templates/METADATA_PERSIST_FAIL.ftl
index 2511b57..f39b281 100644
--- a/core-job/src/main/resources/mail_templates/METADATA_PERSIST_FAIL.ftl
+++ b/core-job/src/main/resources/mail_templates/METADATA_PERSIST_FAIL.ftl
@@ -1,3 +1,20 @@
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
http://git-wip-us.apache.org/repos/asf/kylin/blob/490891e3/external/htrace/pom.xml
----------------------------------------------------------------------
diff --git a/external/htrace/pom.xml b/external/htrace/pom.xml
index 762f853..1c57ddb 100644
--- a/external/htrace/pom.xml
+++ b/external/htrace/pom.xml
@@ -1,27 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
- ~ Copyright (C) 2016 Kyligence Inc. All rights reserved.
- ~
- ~ http://kyligence.io
- ~
- ~ This software is the confidential and proprietary information of
- ~ Kyligence Inc. ("Confidential Information"). You shall not disclose
- ~ such Confidential Information and shall use it only in accordance
- ~ with the terms of the license agreement you entered into with
- ~ Kyligence Inc.
- ~
- ~ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- ~ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- ~ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- ~ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- ~ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- ~ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- ~ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- ~ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- ~ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- ~ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- ~ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- -->
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
http://git-wip-us.apache.org/repos/asf/kylin/blob/490891e3/external/pom.xml
----------------------------------------------------------------------
diff --git a/external/pom.xml b/external/pom.xml
index 8e0a40a..8a43a5d 100644
--- a/external/pom.xml
+++ b/external/pom.xml
@@ -1,27 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
- ~ Copyright (C) 2016 Kyligence Inc. All rights reserved.
- ~
- ~ http://kyligence.io
- ~
- ~ This software is the confidential and proprietary information of
- ~ Kyligence Inc. ("Confidential Information"). You shall not disclose
- ~ such Confidential Information and shall use it only in accordance
- ~ with the terms of the license agreement you entered into with
- ~ Kyligence Inc.
- ~
- ~ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- ~ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- ~ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- ~ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- ~ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- ~ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- ~ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- ~ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- ~ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- ~ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- ~ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- -->
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
http://git-wip-us.apache.org/repos/asf/kylin/blob/490891e3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index dd592bf..7e3cb72 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1213,6 +1213,9 @@
<id>sandbox</id>
<activation>
<activeByDefault>true</activeByDefault>
+ <property>
+ <name>pre-commit</name>
+ </property>
</activation>
<build>
<plugins>
@@ -1420,6 +1423,11 @@
<!-- This profile adds/overrides few features of the 'apache-release'
profile in the parent pom. -->
<id>apache-release</id>
+ <activation>
+ <property>
+ <name>pre-commit</name>
+ </property>
+ </activation>
<build>
<plugins>
<plugin>
@@ -1486,12 +1494,14 @@
<exclude>**/src/test/resources/**</exclude>
<exclude>examples/sample_cube/template/**</exclude>
<exclude>examples/test_case_data/localmeta/**</exclude>
+ <exclude>examples/test_metadata/**</exclude>
<!-- generated files -->
<exclude>**/target/**</exclude>
<exclude>lib/**</exclude>
<exclude>dist/**</exclude>
<exclude>website/**</exclude>
+ <exclude>**/dependency-reduced-pom.xml</exclude>
<!-- text files without comments -->
<exclude>**/*.csv</exclude>
@@ -1556,6 +1566,7 @@
<exclude>**/*.DEF</exclude>
<exclude>**/*.isl</exclude>
<exclude>**/*.isproj</exclude>
+ <exclude>**/*.bmp</exclude>
<!-- protobuf generated -->
<exclude>
[07/50] [abbrv] kylin git commit: Make some hbase config items
configurable at kylin.properties
Posted by li...@apache.org.
Make some hbase config items configurable at kylin.properties
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8e8d71b9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8e8d71b9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8e8d71b9
Branch: refs/heads/sync
Commit: 8e8d71b99a340831512f7ca34cbc0098a190f1f6
Parents: 2ef3d49
Author: nichunen <ch...@kyligence.io>
Authored: Thu Jan 4 19:13:00 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/common/KylinConfigBase.java | 12 ++++++++++++
.../apache/kylin/storage/hbase/HBaseResourceStore.java | 6 +++---
2 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e8d71b9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 4c8a3f8..8efd260 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -358,6 +358,18 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.metadata.check-copy-on-write", "false"));
}
+ public String getHbaseClientScannerTimeoutPeriod() {
+ return getOptional("kylin.metadata.hbase-client-scanner-timeout-period", "10000");
+ }
+
+ public String getHbaseRpcTimeout() {
+ return getOptional("kylin.metadata.hbase-rpc-timeout", "5000");
+ }
+
+ public String getHbaseClientRetriesNumber() {
+ return getOptional("kylin.metadata.hbase-client-retries-number", "1");
+ }
+
// ============================================================================
// DICTIONARY & SNAPSHOT
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/8e8d71b9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
index 433a03c..1f83e3e 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java
@@ -99,9 +99,9 @@ public class HBaseResourceStore extends ResourceStore {
// control timeout for prompt error report
Map<String, String> newParams = new LinkedHashMap<>();
- newParams.put("hbase.client.scanner.timeout.period", "10000");
- newParams.put("hbase.rpc.timeout", "5000");
- newParams.put("hbase.client.retries.number", "1");
+ newParams.put("hbase.client.scanner.timeout.period", kylinConfig.getHbaseClientScannerTimeoutPeriod());
+ newParams.put("hbase.rpc.timeout", kylinConfig.getHbaseRpcTimeout());
+ newParams.put("hbase.client.retries.number", kylinConfig.getHbaseClientRetriesNumber());
newParams.putAll(url.getAllParameters());
return url.copy(newParams);
[26/50] [abbrv] kylin git commit: KYLIN-2760,
change annotation to AclEvaluate call.
Posted by li...@apache.org.
KYLIN-2760, change annotation to AclEvaluate call.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/60caf619
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/60caf619
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/60caf619
Branch: refs/heads/sync
Commit: 60caf619a2daaf06d8b783861990b5a3bd082d81
Parents: d2203f4
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 15 20:19:37 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/service/CubeService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/60caf619/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 8238f5c..186b1ed 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -830,9 +830,9 @@ public class CubeService extends BasicService implements InitializingBean {
return formattedRollingUpCount;
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION')")
public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
+ aclEvaluate.checkProjectAdminPermission(cube.getProject());
return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
}
[25/50] [abbrv] kylin git commit: KYLIN-3157,
enhancement kylin's query timeout.
Posted by li...@apache.org.
KYLIN-3157, enhancement kylin's query timeout.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1f33dd93
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1f33dd93
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1f33dd93
Branch: refs/heads/sync
Commit: 1f33dd93123dd59795a16193a3b5064419f4b532
Parents: 87a0058
Author: Li Yang <li...@apache.org>
Authored: Fri Jan 26 18:39:57 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 13 ++-
.../org/apache/kylin/common/QueryContext.java | 21 +---
.../apache/kylin/storage/StorageFactory.java | 4 +
.../gtrecord/GTCubeStorageQueryBase.java | 3 +-
.../gtrecord/SequentialCubeTupleIterator.java | 5 +-
.../kylin/rest/service/BadQueryDetector.java | 17 ++-
.../apache/kylin/rest/service/QueryService.java | 1 +
.../rest/service/BadQueryDetectorTest.java | 4 +-
.../rest/service/KyilnQueryTimeoutTest.java | 104 +++++++++++++++++++
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 7 +-
10 files changed, 144 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f33dd93/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 8efd260..b053daa 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1266,7 +1266,11 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getBadQueryDefaultDetectIntervalSeconds() {
- return Integer.parseInt(getOptional("kylin.query.badquery-detect-interval", "60"));
+ int time = getQueryTimeoutSeconds() / 2; // half of query timeout
+ if (time == 0) {
+ time = 60; // 60 sec
+ }
+ return time;
}
public boolean getBadQueryPersistentEnabled() {
@@ -1324,7 +1328,12 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getQueryTimeoutSeconds() {
- return Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
+ int time = Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0"));
+ if (time != 0 && time <= 60) {
+ logger.warn("query timeout seconds less than 60 sec, set to 60 sec.");
+ time = 60;
+ }
+ return time;
}
public boolean isPushDownEnabled() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f33dd93/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index cb1b09c..1aa94d3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -45,7 +45,6 @@ public class QueryContext {
}
private long queryStartMillis;
- private long deadline = Long.MAX_VALUE;
private final String queryId;
private String username;
@@ -75,26 +74,10 @@ public class QueryContext {
return queryStartMillis;
}
- public void setDeadline(long timeoutMillis) {
- if (timeoutMillis > 0) {
- deadline = queryStartMillis + timeoutMillis;
- }
- }
-
- public long getDeadline() {
- return deadline;
- }
-
- /**
- * @return millis before deadline
- * @throws KylinTimeoutException if deadline has passed
- */
- public long checkMillisBeforeDeadline() {
- long remain = deadline - System.currentTimeMillis();
- if (remain <= 0) {
+ public void checkMillisBeforeDeadline() {
+ if (Thread.interrupted()) {
throw new KylinTimeoutException("Query timeout");
}
- return remain;
}
public String getQueryId() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f33dd93/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
index 79b93fe..3505708 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
@@ -39,6 +39,10 @@ public class StorageFactory {
return current.get(aware.getStorageType());
}
+ public static void clearCache() {
+ storages.remove();
+ }
+
public static IStorageQuery createQuery(IRealization realization) {
return storage(realization).createQuery(realization);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f33dd93/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index ae1f64f..2f69b76 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -160,8 +160,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
sqlDigest.aggregations, context);
// set whether to aggregate results from multiple partitions
enableStreamAggregateIfBeneficial(cuboid, groupsD, context);
- // set and check query deadline
- QueryContextFacade.current().setDeadline(cubeInstance.getConfig().getQueryTimeoutSeconds() * 1000);
+ // check query deadline
QueryContextFacade.current().checkMillisBeforeDeadline();
// push down having clause filter if possible
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f33dd93/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
index 72417bf..c067e33 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java
@@ -144,10 +144,9 @@ public class SequentialCubeTupleIterator implements ITupleIterator {
if (scanCount++ % 100 == 1) {
QueryContextFacade.current().checkMillisBeforeDeadline();
}
-
- if (++scanCountDelta >= 1000)
+ if (++scanCountDelta >= 1000) {
flushScanCountDelta();
-
+ }
return tupleIterator.next();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f33dd93/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
index 7410c9c..4f7bccf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
@@ -44,7 +44,8 @@ public class BadQueryDetector extends Thread {
private final int alertMB;
private final int alertRunningSec;
private KylinConfig kylinConfig;
- private ArrayList<Notifier> notifiers = new ArrayList<Notifier>();
+ private ArrayList<Notifier> notifiers = new ArrayList<>();
+ private int queryTimeoutSeconds;
public BadQueryDetector() {
super("BadQueryDetector");
@@ -53,17 +54,19 @@ public class BadQueryDetector extends Thread {
this.detectionInterval = kylinConfig.getBadQueryDefaultDetectIntervalSeconds() * 1000L;
this.alertMB = 100;
this.alertRunningSec = kylinConfig.getBadQueryDefaultAlertingSeconds();
+ this.queryTimeoutSeconds = kylinConfig.getQueryTimeoutSeconds();
initNotifiers();
}
- public BadQueryDetector(long detectionInterval, int alertMB, int alertRunningSec) {
+ public BadQueryDetector(long detectionInterval, int alertMB, int alertRunningSec, int queryTimeoutSeconds) {
super("BadQueryDetector");
this.setDaemon(true);
this.detectionInterval = detectionInterval;
this.alertMB = alertMB;
this.alertRunningSec = alertRunningSec;
this.kylinConfig = KylinConfig.getInstanceFromEnv();
+ this.queryTimeoutSeconds = queryTimeoutSeconds;
initNotifiers();
}
@@ -121,6 +124,7 @@ public class BadQueryDetector extends Thread {
notify(badReason, entry);
}
+ @Override
public void run() {
while (true) {
try {
@@ -147,6 +151,8 @@ public class BadQueryDetector extends Thread {
// report if query running long
for (Entry e : entries) {
float runningSec = (float) (now - e.startTime) / 1000;
+ setQueryThreadInterrupted(e, runningSec);
+
if (runningSec >= alertRunningSec) {
notify(BadQueryEntry.ADJ_SLOW, e);
dumpStackTrace(e.thread);
@@ -161,6 +167,13 @@ public class BadQueryDetector extends Thread {
}
}
+ private void setQueryThreadInterrupted(Entry e, float runningSec) {
+ if (queryTimeoutSeconds != 0 && runningSec >= queryTimeoutSeconds) {
+ e.thread.interrupt();
+ logger.error("Trying to cancel query:" + e.thread.getName());
+ }
+ }
+
// log the stack trace of bad query thread for further analysis
private void dumpStackTrace(Thread t) {
int maxStackTraceDepth = kylinConfig.getBadQueryStackTraceDepth();
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f33dd93/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 7b30606..56fab34 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -194,6 +194,7 @@ public class QueryService extends BasicService {
} finally {
String badReason = (ret != null && ret.isPushDown()) ? BadQueryEntry.ADJ_PUSHDOWN : null;
badQueryDetector.queryEnd(Thread.currentThread(), badReason);
+ Thread.interrupted(); //reset if interrupted
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f33dd93/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
index fc18d92..d61dfbe 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
@@ -50,7 +50,7 @@ public class BadQueryDetectorTest extends LocalFileMetadataTestCase {
String mockSql = "select * from just_a_test";
final ArrayList<String[]> alerts = new ArrayList<>();
- BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000, alertMB, alertRunningSec);
+ BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000, alertMB, alertRunningSec, 1000);
badQueryDetector.registerNotifier(new BadQueryDetector.Notifier() {
@Override
public void badQueryFound(String adj, float runningSec, long startTime, String project, String sql, String user, Thread t) {
@@ -72,7 +72,7 @@ public class BadQueryDetectorTest extends LocalFileMetadataTestCase {
badQueryDetector.queryEnd(Thread.currentThread(), BadQueryEntry.ADJ_PUSHDOWN);
}
- badQueryDetector.stop();
+ badQueryDetector.interrupt();
assertEquals(2, alerts.size());
// second check founds a Slow
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f33dd93/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java b/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
new file mode 100644
index 0000000..25ff75b
--- /dev/null
+++ b/server/src/test/java/org/apache/kylin/rest/service/KyilnQueryTimeoutTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.service;
+import java.sql.SQLException;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exceptions.KylinTimeoutException;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.TupleInfo;
+import org.apache.kylin.query.security.QueryACLTestUtil;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.storage.IStorage;
+import org.apache.kylin.storage.IStorageQuery;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.StorageFactory;
+import org.hamcrest.CoreMatchers;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class KyilnQueryTimeoutTest extends LocalFileMetadataTestCase {
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Before
+ public void setUp() {
+ this.createTestMetadata();
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ config.setProperty("kylin.storage.provider.2", MockQueryTimeoutStorage.class.getName());
+ config.setProperty("kylin.storage.default", "2");
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ StorageFactory.clearCache();
+ }
+
+ @Test
+ public void testQueryTimeout() throws SQLException {
+ thrown.expectCause(CoreMatchers.isA(KylinTimeoutException.class));
+ thrown.expectMessage(CoreMatchers.containsString("Kylin query timeout"));
+ StorageFactory.clearCache();
+ BadQueryDetector detector = new BadQueryDetector(100, BadQueryDetector.getSystemAvailMB() * 2, 100, 1);
+ detector.start();
+ SQLRequest request = new SQLRequest();
+ request.setProject("default");
+ request.setSql("select count(*) from STREAMING_TABLE");
+ detector.queryStart(Thread.currentThread(), request, "ADMIN");
+ try {
+ QueryACLTestUtil.mockQuery("default", "select * from STREAMING_TABLE");
+ } finally{
+ detector.queryEnd(Thread.currentThread(), "timeout");
+ detector.interrupt();
+ }
+ }
+
+ public static class MockQueryTimeoutStorage implements IStorage {
+
+ @Override
+ public IStorageQuery createQuery(IRealization realization) {
+ return new MockQueryTimeoutQuery();
+ }
+
+ @Override
+ public <I> I adaptToBuildEngine(Class<I> engineInterface) {
+ return null;
+ }
+ }
+
+ private static class MockQueryTimeoutQuery implements IStorageQuery {
+ @Override
+ public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ try {
+ Thread.sleep(5 * 1000);
+ } catch (InterruptedException e) {
+ throw new KylinTimeoutException("Kylin query timeout");
+ }
+ return null;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f33dd93/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
index 2e82140..634a3cd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseRPC.java
@@ -311,11 +311,8 @@ public abstract class CubeHBaseRPC implements IGTStorage {
coopTimeout = (long) (rpcTimeout * 0.9);
}
- long millisBeforeDeadline = queryContext.checkMillisBeforeDeadline();
- coopTimeout = Math.min(coopTimeout, millisBeforeDeadline);
-
- logger.debug("{} = {} ms, {} ms before deadline, use {} ms as timeout for coprocessor",
- HConstants.HBASE_RPC_TIMEOUT_KEY, rpcTimeout, millisBeforeDeadline, coopTimeout);
+ queryContext.checkMillisBeforeDeadline();
+ logger.debug("{} = {} ms, use {} ms as timeout for coprocessor", HConstants.HBASE_RPC_TIMEOUT_KEY, rpcTimeout, coopTimeout);
return coopTimeout;
}
[32/50] [abbrv] kylin git commit: KYLIN-2909 add freemarker based
EmailTemplateFactory & templates for notification emails
Posted by li...@apache.org.
KYLIN-2909 add freemarker based EmailTemplateFactory & templates for notification emails
Signed-off-by: Zhong <nj...@apache.org>
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7dc18758
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7dc18758
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7dc18758
Branch: refs/heads/sync
Commit: 7dc1875831d225d9968e587599ee274d8d1a93f8
Parents: 8e27449
Author: Zhang Xueyu <co...@mail.ustc.edu.cn>
Authored: Sat Sep 30 13:13:33 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Jan 26 23:09:01 2018 +0800
----------------------------------------------------------------------
core-common/pom.xml | 30 ++
.../kylin/common/util/EmailTemplateEnum.java | 49 +++
.../kylin/common/util/EmailTemplateFactory.java | 98 +++++
.../main/resources/templates/JOB_DISCARD.ftl | 275 ++++++++++++++
.../src/main/resources/templates/JOB_ERROR.ftl | 372 +++++++++++++++++++
.../main/resources/templates/JOB_SUCCEED.ftl | 274 ++++++++++++++
.../templates/METADATA_PERSIST_FAIL.ftl | 179 +++++++++
pom.xml | 6 +
8 files changed, 1283 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7dc18758/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 22733b4..47d16f0 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -50,6 +50,10 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.freemarker</groupId>
+ <artifactId>freemarker</artifactId>
+ </dependency>
<!-- Provided -->
<dependency>
@@ -92,4 +96,30 @@
<version>0.1.2</version>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-templates</id>
+ <phase>process-sources</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/target/classes/templates</outputDirectory>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources/templates</directory>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/7dc18758/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateEnum.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateEnum.java b/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateEnum.java
new file mode 100644
index 0000000..699aa7a
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateEnum.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import com.google.common.base.Strings;
+
+public enum EmailTemplateEnum {
+ JOB_DISCARD("JOB_DISCARD"), JOB_ERROR("JOB_ERROR"), JOB_SUCCEED("JOB_SUCCEED"), //
+ METADATA_PERSIST_FAIL("METADATA_PERSIST_FAIL");
+
+ private final String templateName;
+
+ EmailTemplateEnum(String name) {
+ this.templateName = name;
+ }
+
+ public static EmailTemplateEnum getByName(String name) {
+ if (Strings.isNullOrEmpty(name)) {
+ return null;
+ }
+ for (EmailTemplateEnum value : EmailTemplateEnum.values()) {
+ if (value.templateName.equalsIgnoreCase(name)) {
+ return value;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public String toString() {
+ return templateName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7dc18758/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java b/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java
new file mode 100644
index 0000000..fcf554d
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.StringWriter;
+import java.io.Writer;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import freemarker.template.Configuration;
+import freemarker.template.Template;
+
+public class EmailTemplateFactory {
+
+ private static final Logger logger = LoggerFactory.getLogger(EmailTemplateFactory.class);
+
+ public static final String NA = "NA";
+
+ private static String localHostName;
+ static {
+ try {
+ localHostName = InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (UnknownHostException e) {
+ localHostName = "UNKNOWN";
+ }
+ }
+
+ public static String getLocalHostName() {
+ return localHostName;
+ }
+
+ public static String getEmailTitle(String... titleParts) {
+ StringBuilder sb = new StringBuilder();
+ for (String part : titleParts) {
+ if (sb.length() > 0) {
+ sb.append("-");
+ }
+ sb.append("[" + part + "]");
+ }
+ return sb.toString();
+ }
+
+ private static EmailTemplateFactory instance = new EmailTemplateFactory();
+
+ public static EmailTemplateFactory getInstance() {
+ return instance;
+ }
+
+ private final Configuration configuration;
+
+ private EmailTemplateFactory() {
+ configuration = new Configuration(Configuration.getVersion());
+ configuration.setClassForTemplateLoading(EmailTemplateFactory.class, "/templates");
+ configuration.setDefaultEncoding("UTF-8");
+ }
+
+ public String buildEmailContent(EmailTemplateEnum state, Map<String, Object> root) {
+ try {
+ Template template = getTemplate(state);
+ if (template == null) {
+ return "Cannot find email template for " + state;
+ }
+ try (Writer out = new StringWriter()) {
+ template.process(root, out);
+ return out.toString();
+ }
+ } catch (Throwable e) {
+ return e.getLocalizedMessage();
+ }
+ }
+
+ private Template getTemplate(EmailTemplateEnum state) throws Throwable {
+ if (state == null) {
+ return null;
+ }
+ return configuration.getTemplate(state.toString() + ".ftl");
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7dc18758/core-common/src/main/resources/templates/JOB_DISCARD.ftl
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/templates/JOB_DISCARD.ftl b/core-common/src/main/resources/templates/JOB_DISCARD.ftl
new file mode 100644
index 0000000..fbef3f7
--- /dev/null
+++ b/core-common/src/main/resources/templates/JOB_DISCARD.ftl
@@ -0,0 +1,275 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+<span style="
+ line-height: 1.1;font-size: 18px;">
+ <p style="text-align:left;">Dear Kylin user,</p>
+ <p>It's a pity that the job is discarded.Thank you for using Kylin.</p>
+</span>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h1>
+ <span style="display: inline;
+ background-color: #607D8B;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Discarded </span>
+ </h1>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #f8f8f8">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #eeeeee;
+ border:1px solid #f8f8f8;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #404040;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${job_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #eeeeee;
+ border:1px solid #f8f8f8;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #404040;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${env_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Submitter
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${submitter}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Job Engine
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${job_engine}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Project
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${project_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Cube Name
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${cube_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Source Records Count
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${source_records_count}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Start Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${start_time}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Duration
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${duration}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">MR Waiting Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${mr_waiting}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Last Update Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${last_update_time}
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ </table>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h4 style="font-weight: 500;
+ line-height: 1.1;font-size:18px;">
+ <p>Best Wishes!</p>
+ <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
+ </h4>
+</div>
+</body>
+
+</html>Ò
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7dc18758/core-common/src/main/resources/templates/JOB_ERROR.ftl
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/templates/JOB_ERROR.ftl b/core-common/src/main/resources/templates/JOB_ERROR.ftl
new file mode 100644
index 0000000..6012037
--- /dev/null
+++ b/core-common/src/main/resources/templates/JOB_ERROR.ftl
@@ -0,0 +1,372 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+<span style="
+ line-height: 1.1;font-size: 18px;">
+ <p style="text-align:left;">Dear Kylin user,</p>
+ <p>This cube <strong>failure</strong> may be caused by backend platform being busy, please try again.</p>
+ <p>Thank you for using Kylin and we apologize for the inconvenience.</p>
+</span>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h1>
+ <span style="display: inline;
+ background-color: #d9534f;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Error </span>
+ </h1>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+
+ <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #ebccd1;">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${job_name}
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${env_name}
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Submitter
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${submitter}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Job Engine
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${job_engine}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Project
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${project_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Cube Name
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${cube_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Source Records Count
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${source_records_count}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Start Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${start_time}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Duration
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${duration}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">MR Waiting Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${mr_waiting}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Last Update Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${last_update_time}
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Job Error Details
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Error Step
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${error_step}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ MR Job
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${mr_job_id}
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Logs
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+
+ <pre style="white-space: pre-wrap;">${error_log}</pre>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ </table>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h4 style="font-weight: 500;
+ line-height: 1.1;font-size:18px;">
+ <p>Best Wishes!</p>
+ <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7dc18758/core-common/src/main/resources/templates/JOB_SUCCEED.ftl
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/templates/JOB_SUCCEED.ftl b/core-common/src/main/resources/templates/JOB_SUCCEED.ftl
new file mode 100644
index 0000000..4b443d5
--- /dev/null
+++ b/core-common/src/main/resources/templates/JOB_SUCCEED.ftl
@@ -0,0 +1,274 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+<span style="line-height: 1.1;font-size: 18px;">
+ <p style="text-align:left;">Dear Kylin user,</p>
+ <p>Congratulations! Please feel free to query based on kylin cube.Thank you for using Kylin.</p>
+</span>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h1>
+ <span style="display: inline;
+ background-color: #5cb85c;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Succeed </span>
+ </h1>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #d6e9c6;">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #dff0d8;
+ border:1px solid #d6e9c6;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #3c763d;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${job_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #dff0d8;
+ border:1px solid #d6e9c6;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #3c763d;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${env_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Submitter
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${submitter}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Job Engine
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${job_engine}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Project
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${project_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Cube Name
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${cube_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Source Records Count
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${source_records_count}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Start Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${start_time}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Duration
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${duration}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">MR Waiting Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${mr_waiting}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Last Update Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${last_update_time}
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ </table>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h4 style="font-weight: 500;
+ line-height: 1.1;font-size:18px;">
+ <p>Best Wishes!</p>
+ <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7dc18758/core-common/src/main/resources/templates/METADATA_PERSIST_FAIL.ftl
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/templates/METADATA_PERSIST_FAIL.ftl b/core-common/src/main/resources/templates/METADATA_PERSIST_FAIL.ftl
new file mode 100644
index 0000000..2511b57
--- /dev/null
+++ b/core-common/src/main/resources/templates/METADATA_PERSIST_FAIL.ftl
@@ -0,0 +1,179 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <span style="line-height: 1.1;font-size: 18px;">
+ <p style="text-align:left;">Dear Kylin user,</p>
+ <p>Kylin fails to update the job output due to some hbase issue. Need to ask Hadoop Service Team for help as soon as possible.</p>
+</span>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h1>
+ <span style="display: inline;
+ background-color: #d9534f;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Error </span>
+ </h1>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #ebccd1;">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${job_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${env_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Submitter
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${submitter}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Job Engine
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${job_engine}
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #f2dede;
+ border:1px solid #ebccd1;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #a94442;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ Logs
+ </h4>
+ </td>
+ </tr>
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ <pre style="white-space: pre-wrap;">${error_log}</pre>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ </table>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h4 style="font-weight: 500;
+ line-height: 1.1;font-size:18px;">
+ <p>Best Wishes!</p>
+ <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/7dc18758/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index be2d2bf..dd592bf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -115,6 +115,7 @@
<cors.version>2.5</cors.version>
<tomcat.version>7.0.82</tomcat.version>
<t-digest.version>3.1</t-digest.version>
+ <freemarker.version>2.3.23</freemarker.version>
<!--metric-->
<dropwizard.version>3.1.2</dropwizard.version>
<!-- REST Service, ref https://github.com/spring-projects/spring-boot/blob/v1.3.8.RELEASE/spring-boot-dependencies/pom.xml -->
@@ -651,6 +652,11 @@
<artifactId>commons-validator</artifactId>
<version>${commons-validator.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.freemarker</groupId>
+ <artifactId>freemarker</artifactId>
+ <version>${freemarker.version}</version>
+ </dependency>
<!-- Logging -->
<dependency>
[15/50] [abbrv] kylin git commit: KYLIN-3150 PercentileCounter
maxlength is too small
Posted by li...@apache.org.
KYLIN-3150 PercentileCounter maxlength is too small
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f73219c0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f73219c0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f73219c0
Branch: refs/heads/sync
Commit: f73219c07f5f0d5e01d801aeb274af9cf886f261
Parents: 26bee66
Author: yiming.xu <10...@qq.com>
Authored: Tue Jan 2 21:02:47 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../kylin/measure/percentile/PercentileCounter.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f73219c0/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
index 2b5903c..33433dc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/percentile/PercentileCounter.java
@@ -106,8 +106,16 @@ public class PercentileCounter implements Serializable {
}
public int maxLength() {
- // 10KB for max length
- return 10 * 1024;
+ switch ((int) compression) {
+ case 100:
+ return 16 * 1024;
+ case 1000:
+ return 128 * 1024;
+ case 10000:
+ return 1024 * 1024;
+ default:
+ return 16 * 1024;
+ }
}
public int peekLength(ByteBuffer in) {
[21/50] [abbrv] kylin git commit: minor refactor: avoid exposing
cubedesc's getError public api
Posted by li...@apache.org.
minor refactor: avoid exposing cubedesc's getError public api
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/59d50646
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/59d50646
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/59d50646
Branch: refs/heads/sync
Commit: 59d5064673282ba36783e0796523ad83c2dc5e8c
Parents: 16236ef
Author: Hongbin Ma <ma...@kyligence.io>
Authored: Fri Jan 26 19:13:12 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeDescManager.java | 2 +-
.../org/apache/kylin/cube/CubeInstance.java | 29 ++++++++++--------
.../org/apache/kylin/cube/model/CubeDesc.java | 13 ++------
.../kylin/rest/controller/CubeController.java | 32 ++++++++++++--------
.../apache/kylin/rest/service/CubeService.java | 12 ++++----
5 files changed, 47 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/59d50646/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index cd43f15..1db66e1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -198,7 +198,7 @@ public class CubeDescManager {
postProcessCubeDesc(cubeDesc);
// Check base validation
- if (!cubeDesc.getError().isEmpty()) {
+ if (cubeDesc.isBroken()) {
return cubeDesc;
}
// Semantic validation
http://git-wip-us.apache.org/repos/asf/kylin/blob/59d50646/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index a299203..6395abd 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -6,15 +6,15 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
package org.apache.kylin.cube;
@@ -130,13 +130,13 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
// default constructor for jackson
public CubeInstance() {
}
-
+
public CubeInstance latestCopyForWrite() {
CubeManager mgr = CubeManager.getInstance(config);
CubeInstance latest = mgr.getCube(name); // in case this object is out-of-date
return mgr.copyForWrite(latest);
}
-
+
void init(KylinConfig config) {
CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(descName);
checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", descName, name);
@@ -144,9 +144,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
if (cubeDesc.isBroken()) {
setStatus(RealizationStatusEnum.DESCBROKEN);
logger.error("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), name);
- for (String error : cubeDesc.getError()) {
- logger.error("Error: {}", error);
- }
+ logger.error("Errors: {}", cubeDesc.getErrorsAsString());
} else if (getStatus() == RealizationStatusEnum.DESCBROKEN) {
setStatus(RealizationStatusEnum.DISABLED);
logger.info("cube {} changed from DESCBROKEN to DISABLED", name);
@@ -217,7 +215,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN)
&& segments.isEmpty();
}
-
+
@Override
public String resourceName() {
return name;
@@ -301,7 +299,12 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
public int getCost() {
int countedDimensionNum = getRowKeyColumnCount();
int c = countedDimensionNum * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;
- for (JoinTableDesc join : getModel().getJoinTables()) {
+ DataModelDesc model = getModel();
+ if (model == null) {
+ //in case broken cube
+ return 0;
+ }
+ for (JoinTableDesc join : model.getJoinTables()) {
if (join.getJoin().isInnerJoin())
c += CubeInstance.COST_WEIGHT_INNER_JOIN;
}
@@ -356,7 +359,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
return null;
}
-
+
public CubeSegment[] regetSegments(CubeSegment... segs) {
CubeSegment[] r = new CubeSegment[segs.length];
for (int i = 0; i < segs.length; i++) {
@@ -477,6 +480,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
/**
* Get cuboid level count except base cuboid
+ *
* @return
*/
public int getBuildLevel() {
@@ -562,7 +566,8 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
public SegmentRange autoMergeCubeSegments() throws IOException {
- return segments.autoMergeCubeSegments(needAutoMerge(), getName(), getDescriptor().getAutoMergeTimeRanges(), getDescriptor().getVolatileRange());
+ return segments.autoMergeCubeSegments(needAutoMerge(), getName(), getDescriptor().getAutoMergeTimeRanges(),
+ getDescriptor().getVolatileRange());
}
public Segments calculateToBeSegments(CubeSegment newSegment) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/59d50646/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 2369629..dbd8708 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -82,6 +82,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.base.Joiner;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -1215,16 +1216,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
this.errors.add(message);
}
- public List<String> getError() {
- return this.errors;
- }
-
- public String getErrorMsg() {
- StringBuffer sb = new StringBuffer();
- for (String error : errors) {
- sb.append(error + " ");
- }
- return sb.toString();
+ public String getErrorsAsString() {
+ return Joiner.on("; ").join(errors);
}
public HBaseMappingDesc getHbaseMapping() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/59d50646/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 89c11a8..ff9ecf7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -6,15 +6,15 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
package org.apache.kylin.rest.controller;
@@ -93,7 +93,6 @@ import org.springframework.web.bind.annotation.ResponseBody;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
-import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -209,7 +208,7 @@ public class CubeController extends BasicController {
/**
* Get SQL of a Cube segment
*
- * @param cubeName Cube Name
+ * @param cubeName Cube Name
* @param segmentName Segment Name
* @return
* @throws IOException
@@ -319,7 +318,9 @@ public class CubeController extends BasicController {
}
}
- /** Build/Rebuild a cube segment */
+ /**
+ * Build/Rebuild a cube segment
+ */
@RequestMapping(value = "/{cubeName}/build", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance build(@PathVariable String cubeName, @RequestBody JobBuildRequest req) {
@@ -328,7 +329,9 @@ public class CubeController extends BasicController {
/** Build/Rebuild a cube segment */
- /** Build/Rebuild a cube segment */
+ /**
+ * Build/Rebuild a cube segment
+ */
@RequestMapping(value = "/{cubeName}/rebuild", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance rebuild(@PathVariable String cubeName, @RequestBody JobBuildRequest req) {
@@ -336,7 +339,9 @@ public class CubeController extends BasicController {
req.getBuildType(), req.isForce() || req.isForceMergeEmptySegment());
}
- /** Build/Rebuild a cube segment by source offset */
+ /**
+ * Build/Rebuild a cube segment by source offset
+ */
@RequestMapping(value = "/{cubeName}/build2", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance build2(@PathVariable String cubeName, @RequestBody JobBuildRequest2 req) {
@@ -355,7 +360,9 @@ public class CubeController extends BasicController {
return rebuild2(cubeName, req);
}
- /** Build/Rebuild a cube segment by source offset */
+ /**
+ * Build/Rebuild a cube segment by source offset
+ */
@RequestMapping(value = "/{cubeName}/rebuild2", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance rebuild2(@PathVariable String cubeName, @RequestBody JobBuildRequest2 req) {
@@ -415,7 +422,7 @@ public class CubeController extends BasicController {
/**
* Send a optimize cube segment job
*
- * @param cubeName Cube ID
+ * @param cubeName Cube ID
* @param segmentID for segment to be optimized
*/
@RequestMapping(value = "/{cubeName}/recover_segment_optimize/{segmentID}", method = { RequestMethod.PUT })
@@ -661,8 +668,8 @@ public class CubeController extends BasicController {
throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
}
- if (!desc.getError().isEmpty()) {
- updateRequest(cubeRequest, false, Joiner.on("\n").join(desc.getError()));
+ if (desc.isBroken()) {
+ updateRequest(cubeRequest, false, desc.getErrorsAsString());
return cubeRequest;
}
@@ -959,6 +966,7 @@ public class CubeController extends BasicController {
/**
* Initiate the very beginning of a streaming cube. Will seek the latest offests of each partition from streaming
* source (kafka) and record in the cube descriptor; In the first build job, it will use these offests as the start point.
+ *
* @param cubeName
* @return
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/59d50646/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 186b1ed..c01308d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -203,8 +203,8 @@ public class CubeService extends BasicService implements InitializingBean {
createdDesc = getCubeDescManager().createCubeDesc(desc);
- if (!createdDesc.getError().isEmpty()) {
- throw new BadRequestException(createdDesc.getErrorMsg());
+ if (createdDesc.isBroken()) {
+ throw new BadRequestException(createdDesc.getErrorsAsString());
}
int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc, false);
@@ -633,8 +633,8 @@ public class CubeService extends BasicService implements InitializingBean {
throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT());
}
- if (!desc.getError().isEmpty()) {
- throw new BadRequestException(desc.getErrorMsg());
+ if (desc.isBroken()) {
+ throw new BadRequestException(desc.getErrorsAsString());
}
return desc;
@@ -683,8 +683,8 @@ public class CubeService extends BasicService implements InitializingBean {
throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT());
}
- if (!desc.getError().isEmpty()) {
- throw new BadRequestException(desc.getErrorMsg());
+ if (desc.isBroken()) {
+ throw new BadRequestException(desc.getErrorsAsString());
}
return desc;
[35/50] [abbrv] kylin git commit: KYLIN-2909 refactor to host
templates in each modules
Posted by li...@apache.org.
KYLIN-2909 refactor to host templates in each modules
This closes #91
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/74e3f614
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/74e3f614
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/74e3f614
Branch: refs/heads/sync
Commit: 74e3f614d6a9cc2807865b8fb210836880b8da85
Parents: c96e1b1
Author: lidongsjtu <li...@apache.org>
Authored: Sun Jan 28 19:02:10 2018 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sun Jan 28 19:42:26 2018 +0800
----------------------------------------------------------------------
core-common/pom.xml | 30 +-
.../kylin/common/util/EmailTemplateEnum.java | 49 ---
.../kylin/common/util/EmailTemplateFactory.java | 93 -----
.../kylin/common/util/MailTemplateProvider.java | 77 ++++
.../main/resources/templates/JOB_DISCARD.ftl | 275 --------------
.../src/main/resources/templates/JOB_ERROR.ftl | 372 -------------------
.../main/resources/templates/JOB_SUCCEED.ftl | 274 --------------
.../templates/METADATA_PERSIST_FAIL.ftl | 179 ---------
.../common/util/EmailTemplateFactoryTest.java | 32 --
.../kylin/job/execution/AbstractExecutable.java | 10 +-
.../kylin/job/util/ExecutableStateUtil.java | 38 --
.../kylin/job/util/MailNotificationUtil.java | 81 ++++
.../resources/mail_templates/JOB_DISCARD.ftl | 275 ++++++++++++++
.../main/resources/mail_templates/JOB_ERROR.ftl | 372 +++++++++++++++++++
.../resources/mail_templates/JOB_SUCCEED.ftl | 274 ++++++++++++++
.../mail_templates/METADATA_PERSIST_FAIL.ftl | 179 +++++++++
.../job/util/MailNotificationUtilTest.java | 65 ++++
.../org/apache/kylin/engine/mr/CubingJob.java | 22 +-
18 files changed, 1337 insertions(+), 1360 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 47d16f0..4d12a47 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -64,7 +64,7 @@
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</dependency>
-
+
<!-- Metric & Tracing -->
<dependency>
<groupId>org.apache.kylin</groupId>
@@ -77,7 +77,7 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
-
+
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-jvm</artifactId>
@@ -96,30 +96,4 @@
<version>0.1.2</version>
</dependency>
</dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-templates</id>
- <phase>process-sources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <outputDirectory>${basedir}/target/classes/templates</outputDirectory>
- <resources>
- <resource>
- <directory>${basedir}/src/main/resources/templates</directory>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateEnum.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateEnum.java b/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateEnum.java
deleted file mode 100644
index 699aa7a..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateEnum.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import com.google.common.base.Strings;
-
-public enum EmailTemplateEnum {
- JOB_DISCARD("JOB_DISCARD"), JOB_ERROR("JOB_ERROR"), JOB_SUCCEED("JOB_SUCCEED"), //
- METADATA_PERSIST_FAIL("METADATA_PERSIST_FAIL");
-
- private final String templateName;
-
- EmailTemplateEnum(String name) {
- this.templateName = name;
- }
-
- public static EmailTemplateEnum getByName(String name) {
- if (Strings.isNullOrEmpty(name)) {
- return null;
- }
- for (EmailTemplateEnum value : EmailTemplateEnum.values()) {
- if (value.templateName.equalsIgnoreCase(name)) {
- return value;
- }
- }
- return null;
- }
-
- @Override
- public String toString() {
- return templateName;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java b/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java
deleted file mode 100644
index 2acea5d..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/util/EmailTemplateFactory.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import java.io.StringWriter;
-import java.io.Writer;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Joiner;
-
-import freemarker.template.Configuration;
-import freemarker.template.Template;
-
-public class EmailTemplateFactory {
-
- private static final Logger logger = LoggerFactory.getLogger(EmailTemplateFactory.class);
-
- public static final String NA = "NA";
-
- private static String localHostName;
- static {
- try {
- localHostName = InetAddress.getLocalHost().getCanonicalHostName();
- } catch (UnknownHostException e) {
- localHostName = "UNKNOWN";
- }
- }
-
- public static String getLocalHostName() {
- return localHostName;
- }
-
- public static String getEmailTitle(String... titleParts) {
- return "[" + Joiner.on("]-[").join(titleParts).toString() + "]";
- }
-
- private static EmailTemplateFactory instance = new EmailTemplateFactory();
-
- public static EmailTemplateFactory getInstance() {
- return instance;
- }
-
- private final Configuration configuration;
-
- private EmailTemplateFactory() {
- configuration = new Configuration(Configuration.getVersion());
- configuration.setClassForTemplateLoading(EmailTemplateFactory.class, "/templates");
- configuration.setDefaultEncoding("UTF-8");
- }
-
- public String buildEmailContent(EmailTemplateEnum state, Map<String, Object> root) {
- try {
- Template template = getTemplate(state);
- if (template == null) {
- return "Cannot find email template for " + state;
- }
- try (Writer out = new StringWriter()) {
- template.process(root, out);
- return out.toString();
- }
- } catch (Throwable e) {
- return e.getLocalizedMessage();
- }
- }
-
- private Template getTemplate(EmailTemplateEnum state) throws Throwable {
- if (state == null) {
- return null;
- }
- return configuration.getTemplate(state.toString() + ".ftl");
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-common/src/main/java/org/apache/kylin/common/util/MailTemplateProvider.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MailTemplateProvider.java b/core-common/src/main/java/org/apache/kylin/common/util/MailTemplateProvider.java
new file mode 100644
index 0000000..d403fbc
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MailTemplateProvider.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.common.util;
+
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import freemarker.template.Configuration;
+import freemarker.template.Template;
+
+/**
+ * Use a key to find a template for email.
+ *
+ * The template file is [KEY].ftl file under /mail_templates directory with classloader.
+ */
+public class MailTemplateProvider {
+
+ private static final Logger logger = LoggerFactory.getLogger(MailTemplateProvider.class);
+
+ private static MailTemplateProvider DEFAULT_INSTANCE = new MailTemplateProvider();
+
+ public static MailTemplateProvider getInstance() {
+ return DEFAULT_INSTANCE;
+ }
+
+ private final Configuration configuration;
+
+ private MailTemplateProvider() {
+ configuration = new Configuration(Configuration.getVersion());
+ configuration.setClassForTemplateLoading(MailTemplateProvider.class, "/mail_templates");
+ configuration.setDefaultEncoding("UTF-8");
+ }
+
+ public String buildMailContent(String tplKey, Map<String, Object> data) {
+ try {
+ Template template = getTemplate(tplKey);
+ if (template == null) {
+ return "Cannot find email template for " + tplKey;
+ }
+
+ try (Writer out = new StringWriter()) {
+ template.process(data, out);
+ return out.toString();
+ }
+ } catch (Throwable e) {
+ return e.getLocalizedMessage();
+ }
+ }
+
+ private Template getTemplate(String tplKey) throws Throwable {
+ if (StringUtils.isEmpty(tplKey)) {
+ return null;
+ }
+ return configuration.getTemplate(tplKey + ".ftl");
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-common/src/main/resources/templates/JOB_DISCARD.ftl
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/templates/JOB_DISCARD.ftl b/core-common/src/main/resources/templates/JOB_DISCARD.ftl
deleted file mode 100644
index b00e9aa..0000000
--- a/core-common/src/main/resources/templates/JOB_DISCARD.ftl
+++ /dev/null
@@ -1,275 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml">
-
-<head>
- <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
- <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
-</head>
-
-<style>
- html {
- font-size: 10px;
- }
-
- * {
- box-sizing: border-box;
- }
-
- a:hover,
- a:focus {
- color: #23527c;
- text-decoration: underline;
- }
-
- a:focus {
- outline: 5px auto -webkit-focus-ring-color;
- outline-offset: -2px;
- }
-</style>
-
-<body>
-<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
-<span style="
- line-height: 1.1;font-size: 18px;">
- <p style="text-align:left;">Dear Kylin user,</p>
- <p>It's a pity that the job is discarded. Thank you for using Kylin.</p>
-</span>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
- <h1>
- <span style="display: inline;
- background-color: #607D8B;
- color: #fff;
- line-height: 1;
- font-weight: 700;
- font-size:36px;
- text-align: center;"> Discarded </span>
- </h1>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
- <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #f8f8f8">
-
- <tr>
-
- <td style="padding: 10px 15px;
- background-color: #eeeeee;
- border:1px solid #f8f8f8;">
- <h4 style="margin-top: 0;
- margin-bottom: 0;
- font-size: 16px;
- color: inherit;
- color: #404040;
- font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- ${job_name}
- </h4>
- </td>
- </tr>
-
- <tr>
-
- <td style="padding: 10px 15px;
- background-color: #eeeeee;
- border:1px solid #f8f8f8;">
- <h4 style="margin-top: 0;
- margin-bottom: 0;
- font-size: 16px;
- color: inherit;
- color: #404040;
- font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- ${env_name}
- </h4>
- </td>
- </tr>
-
- <tr>
-
- <td style="padding: 15px;">
- <table cellpadding="0" cellspacing="0" width="100%"
- style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Submitter
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${submitter}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Job Engine
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${job_engine}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Project
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${project_name}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Cube Name
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${cube_name}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">
- Source Records Count
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${source_records_count}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Start Time
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${start_time}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">
- Duration
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${duration}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">MR Waiting Time
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${mr_waiting}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">
- Last Update Time
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${last_update_time}
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
- <h4 style="font-weight: 500;
- line-height: 1.1;font-size:18px;">
- <p>Best Wishes!</p>
- <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
- </h4>
-</div>
-</body>
-
-</html>Ò
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-common/src/main/resources/templates/JOB_ERROR.ftl
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/templates/JOB_ERROR.ftl b/core-common/src/main/resources/templates/JOB_ERROR.ftl
deleted file mode 100644
index 6012037..0000000
--- a/core-common/src/main/resources/templates/JOB_ERROR.ftl
+++ /dev/null
@@ -1,372 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml">
-
-<head>
- <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
- <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
-</head>
-
-<style>
- html {
- font-size: 10px;
- }
-
- * {
- box-sizing: border-box;
- }
-
- a:hover,
- a:focus {
- color: #23527c;
- text-decoration: underline;
- }
-
- a:focus {
- outline: 5px auto -webkit-focus-ring-color;
- outline-offset: -2px;
- }
-</style>
-
-<body>
-<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
-<span style="
- line-height: 1.1;font-size: 18px;">
- <p style="text-align:left;">Dear Kylin user,</p>
- <p>This cube <strong>failure</strong> may be caused by backend platform being busy, please try again.</p>
- <p>Thank you for using Kylin and we apologize for the inconvenience.</p>
-</span>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
- <h1>
- <span style="display: inline;
- background-color: #d9534f;
- color: #fff;
- line-height: 1;
- font-weight: 700;
- font-size:36px;
- text-align: center;"> Error </span>
- </h1>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
-
- <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #ebccd1;">
-
- <tr>
-
- <td style="padding: 10px 15px;
- background-color: #f2dede;
- border:1px solid #ebccd1;">
- <h4 style="margin-top: 0;
- margin-bottom: 0;
- font-size: 16px;
- color: inherit;
- color: #a94442;
- font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- ${job_name}
- </h4>
- </td>
- </tr>
- <tr>
-
- <td style="padding: 10px 15px;
- background-color: #f2dede;
- border:1px solid #ebccd1;">
- <h4 style="margin-top: 0;
- margin-bottom: 0;
- font-size: 16px;
- color: inherit;
- color: #a94442;
- font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- ${env_name}
- </h4>
- </td>
- </tr>
- <tr>
-
- <td style="padding: 15px;">
- <table cellpadding="0" cellspacing="0" width="100%"
- style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Submitter
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${submitter}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Job Engine
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${job_engine}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Project
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${project_name}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Cube Name
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${cube_name}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">
- Source Records Count
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${source_records_count}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Start Time
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${start_time}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">
- Duration
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${duration}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">MR Waiting Time
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${mr_waiting}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">
- Last Update Time
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${last_update_time}
- </td>
- </tr>
- </table>
- </td>
- </tr>
-
- <tr>
-
- <td style="padding: 10px 15px;
- background-color: #f2dede;
- border:1px solid #ebccd1;">
- <h4 style="margin-top: 0;
- margin-bottom: 0;
- font-size: 16px;
- color: inherit;
- color: #a94442;
- font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- Job Error Details
- </h4>
- </td>
- </tr>
- <tr>
-
- <td style="padding: 15px;">
- <table cellpadding="0" cellspacing="0" width="100%"
- style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">
- Error Step
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${error_step}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">
- MR Job
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${mr_job_id}
- </td>
- </tr>
- </table>
- </td>
- </tr>
-
- <tr>
-
- <td style="padding: 10px 15px;
- background-color: #f2dede;
- border:1px solid #ebccd1;">
- <h4 style="margin-top: 0;
- margin-bottom: 0;
- font-size: 16px;
- color: inherit;
- color: #a94442;
- font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- Logs
- </h4>
- </td>
- </tr>
- <tr>
-
- <td style="padding: 15px;">
- <table cellpadding="0" cellspacing="0" width="100%"
- style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- <tr>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
-
- <pre style="white-space: pre-wrap;">${error_log}</pre>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
- <h4 style="font-weight: 500;
- line-height: 1.1;font-size:18px;">
- <p>Best Wishes!</p>
- <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
- </h4>
-</div>
-</body>
-
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-common/src/main/resources/templates/JOB_SUCCEED.ftl
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/templates/JOB_SUCCEED.ftl b/core-common/src/main/resources/templates/JOB_SUCCEED.ftl
deleted file mode 100644
index f1fb45c..0000000
--- a/core-common/src/main/resources/templates/JOB_SUCCEED.ftl
+++ /dev/null
@@ -1,274 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml">
-
-<head>
- <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
- <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
-</head>
-
-<style>
- html {
- font-size: 10px;
- }
-
- * {
- box-sizing: border-box;
- }
-
- a:hover,
- a:focus {
- color: #23527c;
- text-decoration: underline;
- }
-
- a:focus {
- outline: 5px auto -webkit-focus-ring-color;
- outline-offset: -2px;
- }
-</style>
-
-<body>
-<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
-<span style="line-height: 1.1;font-size: 18px;">
- <p style="text-align:left;">Dear Kylin user,</p>
- <p>Congratulations! Please feel free to query based on kylin cube. Thank you for using Kylin.</p>
-</span>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
- <h1>
- <span style="display: inline;
- background-color: #5cb85c;
- color: #fff;
- line-height: 1;
- font-weight: 700;
- font-size:36px;
- text-align: center;"> Succeed </span>
- </h1>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
- <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #d6e9c6;">
-
- <tr>
-
- <td style="padding: 10px 15px;
- background-color: #dff0d8;
- border:1px solid #d6e9c6;">
- <h4 style="margin-top: 0;
- margin-bottom: 0;
- font-size: 16px;
- color: inherit;
- color: #3c763d;
- font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- ${job_name}
- </h4>
- </td>
- </tr>
-
- <tr>
-
- <td style="padding: 10px 15px;
- background-color: #dff0d8;
- border:1px solid #d6e9c6;">
- <h4 style="margin-top: 0;
- margin-bottom: 0;
- font-size: 16px;
- color: inherit;
- color: #3c763d;
- font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- ${env_name}
- </h4>
- </td>
- </tr>
-
- <tr>
-
- <td style="padding: 15px;">
- <table cellpadding="0" cellspacing="0" width="100%"
- style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Submitter
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${submitter}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Job Engine
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${job_engine}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Project
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${project_name}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Cube Name
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${cube_name}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">
- Source Records Count
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${source_records_count}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Start Time
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${start_time}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">
- Duration
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${duration}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">MR Waiting Time
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${mr_waiting}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">
- Last Update Time
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${last_update_time}
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
- <h4 style="font-weight: 500;
- line-height: 1.1;font-size:18px;">
- <p>Best Wishes!</p>
- <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
- </h4>
-</div>
-</body>
-
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-common/src/main/resources/templates/METADATA_PERSIST_FAIL.ftl
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/templates/METADATA_PERSIST_FAIL.ftl b/core-common/src/main/resources/templates/METADATA_PERSIST_FAIL.ftl
deleted file mode 100644
index 2511b57..0000000
--- a/core-common/src/main/resources/templates/METADATA_PERSIST_FAIL.ftl
+++ /dev/null
@@ -1,179 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml">
-
-<head>
- <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
- <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
-</head>
-
-<style>
- html {
- font-size: 10px;
- }
-
- * {
- box-sizing: border-box;
- }
-
- a:hover,
- a:focus {
- color: #23527c;
- text-decoration: underline;
- }
-
- a:focus {
- outline: 5px auto -webkit-focus-ring-color;
- outline-offset: -2px;
- }
-</style>
-
-<body>
-<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- <span style="line-height: 1.1;font-size: 18px;">
- <p style="text-align:left;">Dear Kylin user,</p>
- <p>Kylin fails to update the job output due to some hbase issue. Need to ask Hadoop Service Team for help as soon as possible.</p>
-</span>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
- <h1>
- <span style="display: inline;
- background-color: #d9534f;
- color: #fff;
- line-height: 1;
- font-weight: 700;
- font-size:36px;
- text-align: center;"> Error </span>
- </h1>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
- <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #ebccd1;">
-
- <tr>
-
- <td style="padding: 10px 15px;
- background-color: #f2dede;
- border:1px solid #ebccd1;">
- <h4 style="margin-top: 0;
- margin-bottom: 0;
- font-size: 16px;
- color: inherit;
- color: #a94442;
- font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- ${job_name}
- </h4>
- </td>
- </tr>
-
- <tr>
-
- <td style="padding: 10px 15px;
- background-color: #f2dede;
- border:1px solid #ebccd1;">
- <h4 style="margin-top: 0;
- margin-bottom: 0;
- font-size: 16px;
- color: inherit;
- color: #a94442;
- font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- ${env_name}
- </h4>
- </td>
- </tr>
-
- <tr>
-
- <td style="padding: 15px;">
- <table cellpadding="0" cellspacing="0" width="100%"
- style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Submitter
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${submitter}
- </td>
- </tr>
- <tr>
- <th width="30%" style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- text-align: left;
- font-size: medium;
- font-style: normal;">Job Engine
- </th>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- ${job_engine}
- </td>
- </tr>
- </table>
- </td>
- </tr>
-
- <tr>
-
- <td style="padding: 10px 15px;
- background-color: #f2dede;
- border:1px solid #ebccd1;">
- <h4 style="margin-top: 0;
- margin-bottom: 0;
- font-size: 16px;
- color: inherit;
- color: #a94442;
- font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- Logs
- </h4>
- </td>
- </tr>
- <tr>
-
- <td style="padding: 15px;">
- <table cellpadding="0" cellspacing="0" width="100%"
- style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
- <tr>
- <td style="padding: 8px;
- line-height: 1.42857143;
- vertical-align: top;
- border: 1px solid #ddd;
- font-size: medium;
- font-style: normal;">
- <pre style="white-space: pre-wrap;">${error_log}</pre>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- <hr style="margin-top: 20px;
- margin-bottom: 20px;
- border: 0;
- border-top: 1px solid #eee;">
- <h4 style="font-weight: 500;
- line-height: 1.1;font-size:18px;">
- <p>Best Wishes!</p>
- <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
- </h4>
-</div>
-</body>
-
-</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-common/src/test/java/org/apache/kylin/common/util/EmailTemplateFactoryTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/EmailTemplateFactoryTest.java b/core-common/src/test/java/org/apache/kylin/common/util/EmailTemplateFactoryTest.java
deleted file mode 100644
index 6acbd75..0000000
--- a/core-common/src/test/java/org/apache/kylin/common/util/EmailTemplateFactoryTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.util;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-public class EmailTemplateFactoryTest {
-
- @Test
- public void testGetEmailTitle() {
- String[] titleParts = new String[] { "JOB", "SUCCEED" };
- Assert.assertEquals("[" + titleParts[0] + "]-[" + titleParts[1] + "]",
- EmailTemplateFactory.getEmailTitle(titleParts));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
index f4015bf..1a84871 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java
@@ -29,14 +29,13 @@ import java.util.regex.Matcher;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.EmailTemplateEnum;
-import org.apache.kylin.common.util.EmailTemplateFactory;
import org.apache.kylin.common.util.MailService;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.exception.PersistentException;
import org.apache.kylin.job.impl.threadpool.DefaultContext;
+import org.apache.kylin.job.util.MailNotificationUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -194,13 +193,12 @@ public abstract class AbstractExecutable implements Executable, Idempotent {
dataMap.put("job_name", getName());
dataMap.put("env_name", context.getConfig().getDeployEnv());
dataMap.put("submitter", StringUtil.noBlank(getSubmitter(), "missing submitter"));
- dataMap.put("job_engine", EmailTemplateFactory.getLocalHostName());
+ dataMap.put("job_engine", MailNotificationUtil.getLocalHostName());
dataMap.put("error_log",
Matcher.quoteReplacement(StringUtil.noBlank(exception.getMessage(), "no error message")));
- String content = EmailTemplateFactory.getInstance().buildEmailContent(EmailTemplateEnum.METADATA_PERSIST_FAIL,
- dataMap);
- String title = EmailTemplateFactory.getEmailTitle("METADATA PERSIST", "FAIL",
+ String content = MailNotificationUtil.getMailContent(MailNotificationUtil.METADATA_PERSIST_FAIL, dataMap);
+ String title = MailNotificationUtil.getMailTitle("METADATA PERSIST", "FAIL",
context.getConfig().getDeployEnv());
new MailService(context.getConfig()).sendMail(users, title, content);
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-job/src/main/java/org/apache/kylin/job/util/ExecutableStateUtil.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/util/ExecutableStateUtil.java b/core-job/src/main/java/org/apache/kylin/job/util/ExecutableStateUtil.java
deleted file mode 100644
index 66f806c..0000000
--- a/core-job/src/main/java/org/apache/kylin/job/util/ExecutableStateUtil.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.util;
-
-import org.apache.kylin.common.util.EmailTemplateEnum;
-import org.apache.kylin.job.execution.ExecutableState;
-
-public class ExecutableStateUtil {
-
- public static EmailTemplateEnum getEmailTemplateEnum(ExecutableState state) {
- switch (state) {
- case ERROR:
- return EmailTemplateEnum.JOB_ERROR;
- case DISCARDED:
- return EmailTemplateEnum.JOB_DISCARD;
- case SUCCEED:
- return EmailTemplateEnum.JOB_SUCCEED;
- default:
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-job/src/main/java/org/apache/kylin/job/util/MailNotificationUtil.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/util/MailNotificationUtil.java b/core-job/src/main/java/org/apache/kylin/job/util/MailNotificationUtil.java
new file mode 100644
index 0000000..31c5b20
--- /dev/null
+++ b/core-job/src/main/java/org/apache/kylin/job/util/MailNotificationUtil.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.util;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Map;
+
+import org.apache.kylin.common.util.MailTemplateProvider;
+import org.apache.kylin.job.execution.ExecutableState;
+
+import com.google.common.base.Joiner;
+
+public class MailNotificationUtil {
+ public static final String JOB_ERROR = "JOB_ERROR";
+ public static final String JOB_DISCARD = "JOB_DISCARD";
+ public static final String JOB_SUCCEED = "JOB_SUCCEED";
+ public static final String METADATA_PERSIST_FAIL = "METADATA_PERSIST_FAIL";
+
+ public static final String NA = "NA";
+
+ private static String localHostName;
+
+ static {
+ try {
+ localHostName = InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (UnknownHostException e) {
+ localHostName = "UNKNOWN";
+ }
+ }
+
+ private static String getMailTemplateKey(ExecutableState state) {
+ switch (state) {
+ case ERROR:
+ return JOB_ERROR;
+ case DISCARDED:
+ return JOB_DISCARD;
+ case SUCCEED:
+ return JOB_SUCCEED;
+ default:
+ return null;
+ }
+ }
+
+ public static String getLocalHostName() {
+ return localHostName;
+ }
+
+ public static String getMailContent(ExecutableState state, Map<String, Object> dataMap) {
+ return MailTemplateProvider.getInstance().buildMailContent(MailNotificationUtil.getMailTemplateKey(state),
+ dataMap);
+ }
+
+ public static String getMailContent(String key, Map<String, Object> dataMap) {
+ return MailTemplateProvider.getInstance().buildMailContent(key, dataMap);
+ }
+
+ public static String getMailTitle(String... titleParts) {
+ return "[" + Joiner.on("]-[").join(titleParts) + "]";
+ }
+
+ public static boolean hasMailNotification(ExecutableState state) {
+ return getMailTemplateKey(state) != null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/74e3f614/core-job/src/main/resources/mail_templates/JOB_DISCARD.ftl
----------------------------------------------------------------------
diff --git a/core-job/src/main/resources/mail_templates/JOB_DISCARD.ftl b/core-job/src/main/resources/mail_templates/JOB_DISCARD.ftl
new file mode 100644
index 0000000..9712982
--- /dev/null
+++ b/core-job/src/main/resources/mail_templates/JOB_DISCARD.ftl
@@ -0,0 +1,275 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+ <meta http-equiv="Content-Type" content="Multipart/Alternative; charset=UTF-8"/>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+</head>
+
+<style>
+ html {
+ font-size: 10px;
+ }
+
+ * {
+ box-sizing: border-box;
+ }
+
+ a:hover,
+ a:focus {
+ color: #23527c;
+ text-decoration: underline;
+ }
+
+ a:focus {
+ outline: 5px auto -webkit-focus-ring-color;
+ outline-offset: -2px;
+ }
+</style>
+
+<body>
+<div style="margin-left:5%;margin-right:5%;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+<span style="
+ line-height: 1.1;font-size: 18px;">
+ <p style="text-align:left;">Dear Kylin user,</p>
+ <p>It's a pity that the job is discarded. Thank you for using Kylin.</p>
+</span>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h1>
+ <span style="display: inline;
+ background-color: #607D8B;
+ color: #fff;
+ line-height: 1;
+ font-weight: 700;
+ font-size:36px;
+ text-align: center;"> Discarded </span>
+ </h1>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <table cellpadding="0" cellspacing="0" width="100%" style="border-collapse: collapse;border:1px solid #f8f8f8">
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #eeeeee;
+ border:1px solid #f8f8f8;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #404040;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${job_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 10px 15px;
+ background-color: #eeeeee;
+ border:1px solid #f8f8f8;">
+ <h4 style="margin-top: 0;
+ margin-bottom: 0;
+ font-size: 16px;
+ color: inherit;
+ color: #404040;
+ font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ ${env_name}
+ </h4>
+ </td>
+ </tr>
+
+ <tr>
+
+ <td style="padding: 15px;">
+ <table cellpadding="0" cellspacing="0" width="100%"
+ style="margin-bottom: 20px;border:1 solid #ddd;border-collapse: collapse;font-family: 'Trebuchet MS ', Arial, Helvetica, sans-serif;">
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Submitter
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${submitter}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Job Engine
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${job_engine}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Project
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${project_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Cube Name
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${cube_name}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Source Records Count
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${source_records_count}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">Start Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${start_time}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Duration
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${duration}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">MR Waiting Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${mr_waiting}
+ </td>
+ </tr>
+ <tr>
+ <th width="30%" style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ text-align: left;
+ font-size: medium;
+ font-style: normal;">
+ Last Update Time
+ </th>
+ <td style="padding: 8px;
+ line-height: 1.42857143;
+ vertical-align: top;
+ border: 1px solid #ddd;
+ font-size: medium;
+ font-style: normal;">
+ ${last_update_time}
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ </table>
+ <hr style="margin-top: 20px;
+ margin-bottom: 20px;
+ border: 0;
+ border-top: 1px solid #eee;">
+ <h4 style="font-weight: 500;
+ line-height: 1.1;font-size:18px;">
+ <p>Best Wishes!</p>
+ <p style="margin: 0 0 10px;"><b>Kylin Team</b></p>
+ </h4>
+</div>
+</body>
+
+</html>
\ No newline at end of file
[18/50] [abbrv] kylin git commit: Deep copy segment when duplicate
cube instance
Posted by li...@apache.org.
Deep copy segment when duplicate cube instance
(cherry picked from commit 4d5cf906fbe4cd840e2daf7333be8c52c7ccfb29)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73554fa7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73554fa7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73554fa7
Branch: refs/heads/sync
Commit: 73554fa71cd4bb519b19dab3c9e4a19e19d33d1e
Parents: ce1775a
Author: Yifan Zhang <ev...@gmail.com>
Authored: Wed Jan 24 18:45:34 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 9 +++++++-
.../org/apache/kylin/cube/CubeInstanceTest.java | 22 +++++++++++++++++---
2 files changed, 27 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/73554fa7/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 6395abd..27918bc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -191,6 +191,9 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
public CubeDesc getDescriptor() {
+ if (config == null) {
+ return null;
+ }
return CubeDescManager.getInstance(config).getCubeDesc(descName);
}
@@ -297,6 +300,10 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
@Override
public int getCost() {
+ if (getDescriptor() == null) {
+ //in case not initialized
+ return 0;
+ }
int countedDimensionNum = getRowKeyColumnCount();
int c = countedDimensionNum * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;
DataModelDesc model = getModel();
@@ -348,7 +355,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
public void setSegments(Segments segments) {
- this.segments = segments;
+ this.segments = new Segments<>(segments);
}
public CubeSegment getSegmentById(String segmentId) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/73554fa7/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
index 9e517b9..dd3b12b 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
@@ -18,17 +18,18 @@
package org.apache.kylin.cube;
-import static org.junit.Assert.assertNotNull;
-
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.cube.cuboid.TreeCuboidScheduler;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -56,6 +57,21 @@ public class CubeInstanceTest {
for (Long cuboid : cuboids) {
System.out.println(cuboid + ":" + cuboidsWithRowCnt.get(cuboid));
}
- assertNotNull(cuboidsWithRowCnt.get(255L));
+ Assert.assertNotNull(cuboidsWithRowCnt.get(255L));
+ }
+
+ @Test
+ public void copyCubeSegmentTest() {
+ int origSegCount = cubeInstance.getSegments().size();
+ CubeInstance newCubeInstance = CubeInstance.getCopyOf(cubeInstance);
+
+ CubeSegment mockSeg = new CubeSegment();
+ mockSeg.setUuid(UUID.randomUUID().toString());
+ mockSeg.setStorageLocationIdentifier(UUID.randomUUID().toString());
+ mockSeg.setStatus(SegmentStatusEnum.READY);
+ newCubeInstance.getSegments().add(mockSeg);
+
+ Assert.assertEquals(origSegCount, cubeInstance.getSegments().size());
+ Assert.assertEquals(origSegCount + 1, newCubeInstance.getSegments().size());
}
}
[49/50] [abbrv] kylin git commit: minor,
tell prepared SQL in log summary
Posted by li...@apache.org.
minor, tell prepared SQL in log summary
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4ade239f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4ade239f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4ade239f
Branch: refs/heads/sync
Commit: 4ade239fd0ba027fda70b2e61f2a510ee31fbaea
Parents: da14ee9
Author: Yifan Zhang <ev...@gmail.com>
Authored: Wed Jan 31 00:03:31 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Feb 4 08:45:09 2018 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/service/QueryService.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4ade239f/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 841cad9..bbf71c7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -323,6 +323,7 @@ public class QueryService extends BasicService {
stringBuilder.append("Hit Exception Cache: ").append(response.isHitExceptionCache()).append(newLine);
stringBuilder.append("Storage cache used: ").append(storageCacheUsed).append(newLine);
stringBuilder.append("Is Query Push-Down: ").append(isPushDown).append(newLine);
+ stringBuilder.append("Is Prepare: ").append(BackdoorToggles.getPrepareOnly()).append(newLine);
stringBuilder.append("Trace URL: ").append(response.getTraceUrl()).append(newLine);
stringBuilder.append("Message: ").append(response.getExceptionMessage()).append(newLine);
stringBuilder.append("==========================[QUERY]===============================").append(newLine);
[17/50] [abbrv] kylin git commit: KYLIN-1664 Revert the incorrect
change.
Posted by li...@apache.org.
KYLIN-1664 Revert the incorrect change.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/adf645e3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/adf645e3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/adf645e3
Branch: refs/heads/sync
Commit: adf645e319b5f49708bd6a540a0a980a7339885d
Parents: 9dfd8b9
Author: tttMelody <24...@qq.com>
Authored: Mon Jan 15 15:05:27 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../kylin/rest/service/AdminServiceTest.java | 35 +++++++++++++-------
1 file changed, 23 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/adf645e3/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
index 3ee14e0..ce46536 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
@@ -46,17 +46,28 @@ public class AdminServiceTest extends ServiceTestBase {
String path = Thread.currentThread().getContextClassLoader().getResource("kylin.properties").getPath();
KylinConfig.setKylinConfigThreadLocal(KylinConfig.createInstanceFromUri(path));
- String publicConfig = adminService.getPublicConfig();
-
- Assert.assertFalse(publicConfig.contains("kylin.metadata.data-model-manager-impl"));
- Assert.assertFalse(publicConfig.contains("kylin.dictionary.use-forest-trie"));
- Assert.assertFalse(publicConfig.contains("kylin.cube.segment-advisor"));
- Assert.assertFalse(publicConfig.contains("kylin.job.use-remote-cli"));
- Assert.assertFalse(publicConfig.contains("kylin.job.scheduler.provider"));
- Assert.assertFalse(publicConfig.contains("kylin.engine.mr.job-jar"));
- Assert.assertFalse(publicConfig.contains("kylin.engine.spark.sanity-check-enabled"));
- Assert.assertFalse(publicConfig.contains("kylin.storage.provider"));
- Assert.assertFalse(publicConfig.contains("kylin.query.convert-create-table-to-with"));
- Assert.assertFalse(publicConfig.contains("kylin.server.init-tasks"));
+ String expected = "kylin.web.link-streaming-guide=http://kylin.apache.org/\n" +
+ "kylin.web.contact-mail=\n" +
+ "kylin.query.cache-enabled=true\n" +
+ "kylin.web.link-diagnostic=\n" +
+ "kylin.web.help.length=4\n" +
+ "kylin.web.timezone=GMT+8\n" +
+ "kylin.server.external-acl-provider=\n" +
+ "kylin.storage.default=2\n" +
+ "kylin.web.help=\n" +
+ "kylin.web.export-allow-other=true\n" +
+ "kylin.web.link-hadoop=\n" +
+ "kylin.web.hide-measures=RAW\n" +
+ "kylin.htrace.show-gui-trace-toggle=false\n" +
+ "kylin.web.export-allow-admin=true\n" +
+ "kylin.env=QA\n" +
+ "kylin.web.hive-limit=20\n" +
+ "kylin.engine.default=2\n" +
+ "kylin.web.help.3=onboard|Cube Design Tutorial|\n" +
+ "kylin.web.help.2=tableau|Tableau Guide|\n" +
+ "kylin.web.help.1=odbc|ODBC Driver|\n" +
+ "kylin.web.help.0=start|Getting Started|\n" +
+ "kylin.security.profile=testing\n";
+ Assert.assertEquals(expected, adminService.getPublicConfig());
}
}
[27/50] [abbrv] kylin git commit: KYLIN-3164 HBase connection must be
closed when clearing connection pool
Posted by li...@apache.org.
KYLIN-3164 HBase connection must be closed when clearing connection pool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9dfd8b9c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9dfd8b9c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9dfd8b9c
Branch: refs/heads/sync
Commit: 9dfd8b9c8a5ad4b185f9855b7ef167087f06d2a1
Parents: baf7133
Author: Li Yang <li...@apache.org>
Authored: Mon Jan 15 13:25:56 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../kylin/storage/hbase/HBaseConnection.java | 38 +++++++++++++++-----
1 file changed, 29 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9dfd8b9c/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 16bd20d..0c2fb04 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -20,8 +20,10 @@ package org.apache.kylin.storage.hbase;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -74,14 +76,11 @@ public class HBaseConnection {
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
- closeCoprocessorPool();
-
- for (Connection conn : connPool.values()) {
- try {
- conn.close();
- } catch (IOException e) {
- logger.error("error closing hbase connection " + conn, e);
- }
+ try {
+ closeCoprocessorPool();
+ closeAndRestConnPool().join();
+ } catch (InterruptedException e) {
+ logger.error("", e);
}
}
});
@@ -128,9 +127,30 @@ public class HBaseConnection {
coprocessorPool.shutdownNow();
}
}
+
+ private static Thread closeAndRestConnPool() {
+ final List<Connection> copy = new ArrayList<>(connPool.values());
+ connPool.clear();
+
+ Thread t = new Thread() {
+ public void run() {
+ logger.info("Closing HBase connections...");
+ for (Connection conn : copy) {
+ try {
+ conn.close();
+ } catch (Exception e) {
+ logger.error("error closing hbase connection " + conn, e);
+ }
+ }
+ }
+ };
+ t.setName("close-hbase-conn");
+ t.start();
+ return t;
+ }
public static void clearConnCache() {
- connPool.clear();
+ closeAndRestConnPool();
}
public static Configuration getCurrentHBaseConfiguration() {
[13/50] [abbrv] kylin git commit: minor,
restore OLAPToEnumerableConverter to original
Posted by li...@apache.org.
minor, restore OLAPToEnumerableConverter to original
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a4acb4d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a4acb4d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a4acb4d5
Branch: refs/heads/sync
Commit: a4acb4d53ef74f8e59f46f550143c599c546ff30
Parents: f73219c
Author: Roger Shi <ro...@hotmail.com>
Authored: Wed Jan 3 22:21:28 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../apache/calcite/plan/OLAPRelMdRowCount.java | 174 ----------------
.../calcite/plan/OLAPRelMetadataProvider.java | 39 ----
.../org/apache/calcite/plan/RelOptCluster.java | 199 -------------------
.../src/test/resources/query/sql/query110.sql | 58 ++++++
.../relnode/OLAPToEnumerableConverter.java | 2 +-
5 files changed, 59 insertions(+), 413 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a4acb4d5/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java b/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
deleted file mode 100644
index 751561f..0000000
--- a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMdRowCount.java
+++ /dev/null
@@ -1,174 +0,0 @@
-package org.apache.calcite.plan;
-
-import org.apache.calcite.adapter.enumerable.EnumerableLimit;
-import org.apache.calcite.plan.volcano.RelSubset;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.SingleRel;
-import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.Calc;
-import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.Intersect;
-import org.apache.calcite.rel.core.Join;
-import org.apache.calcite.rel.core.Minus;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.SemiJoin;
-import org.apache.calcite.rel.core.Sort;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.core.Union;
-import org.apache.calcite.rel.core.Values;
-import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMdRowCount;
-import org.apache.calcite.rel.metadata.RelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.util.BuiltInMethod;
-
-public class OLAPRelMdRowCount extends RelMdRowCount {
- public static final RelMetadataProvider SOURCE = ReflectiveRelMetadataProvider
- .reflectiveSource(BuiltInMethod.ROW_COUNT.method, new OLAPRelMdRowCount());
-
- //avoid case like sql/query60.sql, which will generate plan:
- /*
- EnumerableLimit(fetch=[3])
- OLAPToEnumerableConverter
- OLAPAggregateRel(group=[{0}], SUM_PRICE=[SUM($1)], CNT_1=[COUNT()], TOTAL_ITEMS=[SUM($2)])
- */
- private boolean shouldIntercept(RelNode rel) {
- for (RelNode input : rel.getInputs()) {
- if (input instanceof RelSubset) {
- RelSubset relSubset = (RelSubset) input;
- if (relSubset.getBest() != null
- && relSubset.getBest().getClass().getCanonicalName().endsWith("OLAPToEnumerableConverter")) {
- return true;
- }
- }
- }
- return false;
- }
-
- @Override
- public Double getRowCount(RelNode rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(RelSubset subset, RelMetadataQuery mq) {
- if (shouldIntercept(subset))
- return 1E10;
-
- return super.getRowCount(subset, mq);
- }
-
- @Override
- public Double getRowCount(Union rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Intersect rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Minus rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Filter rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Calc rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Project rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Sort rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(EnumerableLimit rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(SingleRel rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Join rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(SemiJoin rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Aggregate rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(TableScan rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-
- @Override
- public Double getRowCount(Values rel, RelMetadataQuery mq) {
- if (shouldIntercept(rel))
- return 1E10;
-
- return super.getRowCount(rel, mq);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a4acb4d5/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMetadataProvider.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMetadataProvider.java b/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMetadataProvider.java
deleted file mode 100644
index ad944db..0000000
--- a/atopcalcite/src/main/java/org/apache/calcite/plan/OLAPRelMetadataProvider.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package org.apache.calcite.plan;
-
-import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMdAllPredicates;
-import org.apache.calcite.rel.metadata.RelMdCollation;
-import org.apache.calcite.rel.metadata.RelMdColumnOrigins;
-import org.apache.calcite.rel.metadata.RelMdColumnUniqueness;
-import org.apache.calcite.rel.metadata.RelMdDistinctRowCount;
-import org.apache.calcite.rel.metadata.RelMdDistribution;
-import org.apache.calcite.rel.metadata.RelMdExplainVisibility;
-import org.apache.calcite.rel.metadata.RelMdExpressionLineage;
-import org.apache.calcite.rel.metadata.RelMdMaxRowCount;
-import org.apache.calcite.rel.metadata.RelMdMemory;
-import org.apache.calcite.rel.metadata.RelMdMinRowCount;
-import org.apache.calcite.rel.metadata.RelMdNodeTypes;
-import org.apache.calcite.rel.metadata.RelMdParallelism;
-import org.apache.calcite.rel.metadata.RelMdPercentageOriginalRows;
-import org.apache.calcite.rel.metadata.RelMdPopulationSize;
-import org.apache.calcite.rel.metadata.RelMdPredicates;
-import org.apache.calcite.rel.metadata.RelMdSelectivity;
-import org.apache.calcite.rel.metadata.RelMdSize;
-import org.apache.calcite.rel.metadata.RelMdTableReferences;
-import org.apache.calcite.rel.metadata.RelMdUniqueKeys;
-
-import com.google.common.collect.ImmutableList;
-
-public class OLAPRelMetadataProvider extends ChainedRelMetadataProvider {
- public static final OLAPRelMetadataProvider INSTANCE = new OLAPRelMetadataProvider();
-
- public OLAPRelMetadataProvider() {
- super(ImmutableList.of(RelMdPercentageOriginalRows.SOURCE, RelMdColumnOrigins.SOURCE,
- RelMdExpressionLineage.SOURCE, RelMdTableReferences.SOURCE, RelMdNodeTypes.SOURCE,
- OLAPRelMdRowCount.SOURCE, RelMdMaxRowCount.SOURCE, RelMdMinRowCount.SOURCE, RelMdUniqueKeys.SOURCE,
- RelMdColumnUniqueness.SOURCE, RelMdPopulationSize.SOURCE, RelMdSize.SOURCE, RelMdParallelism.SOURCE,
- RelMdDistribution.SOURCE, RelMdMemory.SOURCE, RelMdDistinctRowCount.SOURCE, RelMdSelectivity.SOURCE,
- RelMdExplainVisibility.SOURCE, RelMdPredicates.SOURCE, RelMdAllPredicates.SOURCE,
- RelMdCollation.SOURCE));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a4acb4d5/atopcalcite/src/main/java/org/apache/calcite/plan/RelOptCluster.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/plan/RelOptCluster.java b/atopcalcite/src/main/java/org/apache/calcite/plan/RelOptCluster.java
deleted file mode 100644
index 62a3ef5..0000000
--- a/atopcalcite/src/main/java/org/apache/calcite/plan/RelOptCluster.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.calcite.plan;
-
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.CorrelationId;
-import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider;
-import org.apache.calcite.rel.metadata.MetadataFactory;
-import org.apache.calcite.rel.metadata.MetadataFactoryImpl;
-import org.apache.calcite.rel.metadata.RelMetadataProvider;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rex.RexBuilder;
-import org.apache.calcite.rex.RexNode;
-
-import com.google.common.base.Preconditions;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/*
- * OVERRIDE POINT:
- * - grep OLAPRelMetadataProvider();
- */
-/**
- * An environment for related relational expressions during the
- * optimization of a query.
- */
-public class RelOptCluster {
- //~ Instance fields --------------------------------------------------------
-
- private final RelDataTypeFactory typeFactory;
- private final RelOptPlanner planner;
- private final AtomicInteger nextCorrel;
- private final Map<String, RelNode> mapCorrelToRel;
- private RexNode originalExpression;
- private final RexBuilder rexBuilder;
- private RelMetadataProvider metadataProvider;
- private MetadataFactory metadataFactory;
- private final RelTraitSet emptyTraitSet;
- private RelMetadataQuery mq;
-
- //~ Constructors -----------------------------------------------------------
-
- /**
- * Creates a cluster.
- */
- @Deprecated // to be removed before 2.0
- RelOptCluster(
- RelOptQuery query,
- RelOptPlanner planner,
- RelDataTypeFactory typeFactory,
- RexBuilder rexBuilder) {
- this(planner, typeFactory, rexBuilder, query.nextCorrel,
- query.mapCorrelToRel);
- }
-
- /**
- * Creates a cluster.
- *
- * <p>For use only from {@link #create} and {@link RelOptQuery}.
- */
- RelOptCluster(RelOptPlanner planner, RelDataTypeFactory typeFactory,
- RexBuilder rexBuilder, AtomicInteger nextCorrel,
- Map<String, RelNode> mapCorrelToRel) {
- this.nextCorrel = nextCorrel;
- this.mapCorrelToRel = mapCorrelToRel;
- this.planner = Preconditions.checkNotNull(planner);
- this.typeFactory = Preconditions.checkNotNull(typeFactory);
- this.rexBuilder = rexBuilder;
- this.originalExpression = rexBuilder.makeLiteral("?");
-
- // set up a default rel metadata provider,
- // giving the planner first crack at everything
- setMetadataProvider(OLAPRelMetadataProvider.INSTANCE);
- this.emptyTraitSet = planner.emptyTraitSet();
- assert emptyTraitSet.size() == planner.getRelTraitDefs().size();
- }
-
- /** Creates a cluster. */
- public static RelOptCluster create(RelOptPlanner planner,
- RexBuilder rexBuilder) {
- return new RelOptCluster(planner, rexBuilder.getTypeFactory(),
- rexBuilder, new AtomicInteger(0), new HashMap<String, RelNode>());
- }
-
- //~ Methods ----------------------------------------------------------------
-
- @Deprecated // to be removed before 2.0
- public RelOptQuery getQuery() {
- return new RelOptQuery(planner, nextCorrel, mapCorrelToRel);
- }
-
- @Deprecated // to be removed before 2.0
- public RexNode getOriginalExpression() {
- return originalExpression;
- }
-
- @Deprecated // to be removed before 2.0
- public void setOriginalExpression(RexNode originalExpression) {
- this.originalExpression = originalExpression;
- }
-
- public RelOptPlanner getPlanner() {
- return planner;
- }
-
- public RelDataTypeFactory getTypeFactory() {
- return typeFactory;
- }
-
- public RexBuilder getRexBuilder() {
- return rexBuilder;
- }
-
- public RelMetadataProvider getMetadataProvider() {
- return metadataProvider;
- }
-
- /**
- * Overrides the default metadata provider for this cluster.
- *
- * @param metadataProvider custom provider
- */
- public void setMetadataProvider(RelMetadataProvider metadataProvider) {
- this.metadataProvider = metadataProvider;
- this.metadataFactory = new MetadataFactoryImpl(metadataProvider);
- }
-
- public MetadataFactory getMetadataFactory() {
- return metadataFactory;
- }
-
- /** Returns the current RelMetadataQuery.
- *
- * <p>This method might be changed or moved in future.
- * If you have a {@link RelOptRuleCall} available,
- * for example if you are in a {@link RelOptRule#onMatch(RelOptRuleCall)}
- * method, then use {@link RelOptRuleCall#getMetadataQuery()} instead. */
- public RelMetadataQuery getMetadataQuery() {
- if (mq == null) {
- mq = RelMetadataQuery.instance();
- }
- return mq;
- }
-
- /**
- * Should be called whenever the current {@link RelMetadataQuery} becomes
- * invalid. Typically invoked from {@link RelOptRuleCall#transformTo}.
- */
- public void invalidateMetadataQuery() {
- mq = null;
- }
-
- /**
- * Constructs a new id for a correlating variable. It is unique within the
- * whole query.
- */
- public CorrelationId createCorrel() {
- return new CorrelationId(nextCorrel.getAndIncrement());
- }
-
- /** Returns the default trait set for this cluster. */
- public RelTraitSet traitSet() {
- return emptyTraitSet;
- }
-
- /** @deprecated For {@code traitSetOf(t1, t2)},
- * use {@link #traitSet}().replace(t1).replace(t2). */
- @Deprecated // to be removed before 2.0
- public RelTraitSet traitSetOf(RelTrait... traits) {
- RelTraitSet traitSet = emptyTraitSet;
- for (RelTrait trait : traits) {
- traitSet = traitSet.replace(trait);
- }
- return traitSet;
- }
-
- public RelTraitSet traitSetOf(RelTrait trait) {
- return emptyTraitSet.replace(trait);
- }
-}
-
-// End RelOptCluster.java
http://git-wip-us.apache.org/repos/asf/kylin/blob/a4acb4d5/kylin-it/src/test/resources/query/sql/query110.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query110.sql b/kylin-it/src/test/resources/query/sql/query110.sql
new file mode 100644
index 0000000..07cd3b2
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql/query110.sql
@@ -0,0 +1,58 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+SELECT TEST_KYLIN_FACT.CAL_DT as TEST_KYLIN_FACT_CAL_DT,
+ TEST_KYLIN_FACT.ORDER_ID as TEST_KYLIN_FACT_ORDER_ID,
+ TEST_ORDER.TEST_DATE_ENC as TEST_ORDER_TEST_DATE_ENC,
+ TEST_ORDER.TEST_TIME_ENC as TEST_ORDER_TEST_TIME_ENC,
+ TEST_KYLIN_FACT.LEAF_CATEG_ID as TEST_KYLIN_FACT_LEAF_CATEG_ID,
+ TEST_KYLIN_FACT.LSTG_SITE_ID as TEST_KYLIN_FACT_LSTG_SITE_ID,
+ TEST_CATEGORY_GROUPINGS.META_CATEG_NAME as TEST_CATEGORY_GROUPINGS_META_CATEG_NAME,
+ TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME as TEST_CATEGORY_GROUPINGS_CATEG_LVL2_NAME,
+ TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME as TEST_CATEGORY_GROUPINGS_CATEG_LVL3_NAME,
+ TEST_KYLIN_FACT.LSTG_FORMAT_NAME as TEST_KYLIN_FACT_LSTG_FORMAT_NAME,
+ TEST_KYLIN_FACT.SLR_SEGMENT_CD as TEST_KYLIN_FACT_SLR_SEGMENT_CD,
+ TEST_KYLIN_FACT.SELLER_ID as TEST_KYLIN_FACT_SELLER_ID,
+ SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL as SELLER_ACCOUNT_ACCOUNT_BUYER_LEVEL,
+ SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL as SELLER_ACCOUNT_ACCOUNT_SELLER_LEVEL,
+ SELLER_ACCOUNT.ACCOUNT_COUNTRY as SELLER_ACCOUNT_ACCOUNT_COUNTRY,
+ SELLER_COUNTRY.NAME as SELLER_COUNTRY_NAME,
+ TEST_ORDER.BUYER_ID as TEST_ORDER_BUYER_ID,
+ BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL as BUYER_ACCOUNT_ACCOUNT_BUYER_LEVEL,
+ BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL as BUYER_ACCOUNT_ACCOUNT_SELLER_LEVEL,
+ BUYER_ACCOUNT.ACCOUNT_COUNTRY as BUYER_ACCOUNT_ACCOUNT_COUNTRY,
+ year(TEST_KYLIN_FACT.CAL_DT) as TEST_KYLIN_FACT_DEAL_YEAR
+ FROM "DEFAULT".TEST_KYLIN_FACT as TEST_KYLIN_FACT
+ LEFT JOIN "DEFAULT".TEST_ORDER as TEST_ORDER
+ ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
+ LEFT JOIN EDW.TEST_CAL_DT as TEST_CAL_DT
+ ON TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT
+ LEFT JOIN "DEFAULT".TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
+ ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
+ LEFT JOIN EDW.TEST_SITES as TEST_SITES
+ ON TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_SITES.SITE_ID
+ LEFT JOIN EDW.TEST_SELLER_TYPE_DIM as TEST_SELLER_TYPE_DIM
+ ON TEST_KYLIN_FACT.SLR_SEGMENT_CD = TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD
+ LEFT JOIN "DEFAULT".TEST_ACCOUNT as SELLER_ACCOUNT
+ ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
+ LEFT JOIN "DEFAULT".TEST_ACCOUNT as BUYER_ACCOUNT
+ ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
+ LEFT JOIN "DEFAULT".TEST_COUNTRY as SELLER_COUNTRY
+ ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
+ LEFT JOIN "DEFAULT".TEST_COUNTRY as BUYER_COUNTRY
+ ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
+ WHERE 1=1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/a4acb4d5/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
index 697827f..89c9041 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPToEnumerableConverter.java
@@ -58,7 +58,7 @@ public class OLAPToEnumerableConverter extends ConverterImpl implements Enumerab
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
// huge cost to ensure OLAPToEnumerableConverter only appears once in rel tree
- return planner.getCostFactory().makeCost(1E10, 0, 0);
+ return planner.getCostFactory().makeCost(1E100, 0, 0);
}
@Override
[46/50] [abbrv] kylin git commit: KYLIN-3216
Posted by li...@apache.org.
KYLIN-3216
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c7718334
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c7718334
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c7718334
Branch: refs/heads/sync
Commit: c771833434523aecef2e6f33d9055a92a2c6f7cb
Parents: 490891e
Author: xingpeng1 <xi...@zte.com.cn>
Authored: Wed Jan 31 18:16:35 2018 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Thu Feb 1 09:43:13 2018 +0800
----------------------------------------------------------------------
build/bin/check-env.sh | 20 +++++++++++++++++---
1 file changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c7718334/build/bin/check-env.sh
----------------------------------------------------------------------
diff --git a/build/bin/check-env.sh b/build/bin/check-env.sh
index 7534d12..cd745d4 100755
--- a/build/bin/check-env.sh
+++ b/build/bin/check-env.sh
@@ -62,8 +62,22 @@ then
quit "Failed to create $WORKING_DIR. Please make sure the user has right to access $WORKING_DIR"
fi
-hadoop ${hadoop_conf_param} fs -mkdir -p $WORKING_DIR/spark-history
-if [ $? != 0 ]
+SPARK_EVENTLOG_DIR=`bash $KYLIN_HOME/bin/get-properties.sh kylin.engine.spark-conf.spark.eventLog.dir`
+if [ -n "$SPARK_EVENTLOG_DIR" ]
+then
+ hadoop ${hadoop_conf_param} fs -mkdir -p $SPARK_EVENTLOG_DIR
+ if [ $? != 0 ]
+ then
+ quit "Failed to create $SPARK_EVENTLOG_DIR. Please make sure the user has right to access $SPARK_EVENTLOG_DIR"
+ fi
+fi
+
+SPARK_HISTORYLOG_DIR=`bash $KYLIN_HOME/bin/get-properties.sh kylin.engine.spark-conf.spark.history.fs.logDirectory`
+if [ -n "$SPARK_HISTORYLOG_DIR" ]
then
- quit "Failed to create $WORKING_DIR/spark-history. Please make sure the user has right to access $WORKING_DIR"
+ hadoop ${hadoop_conf_param} fs -mkdir -p $SPARK_HISTORYLOG_DIR
+ if [ $? != 0 ]
+ then
+ quit "Failed to create $SPARK_HISTORYLOG_DIR. Please make sure the user has right to access $SPARK_HISTORYLOG_DIR"
+ fi
fi
\ No newline at end of file
[23/50] [abbrv] kylin git commit: minor,
refine basic tuple filter class
Posted by li...@apache.org.
minor, refine basic tuple filter class
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c2b69155
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c2b69155
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c2b69155
Branch: refs/heads/sync
Commit: c2b69155215f72a5f78bf65995390431ff424c1e
Parents: 1f33dd9
Author: Roger Shi <ro...@hotmail.com>
Authored: Mon Jan 8 17:57:57 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../metadata/filter/LogicalTupleFilter.java | 26 ++++++++
.../kylin/metadata/filter/TupleFilter.java | 43 +++++++++++++
.../kylin/metadata/filter/TupleFilterTest.java | 65 ++++++++++++++++++++
3 files changed, 134 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c2b69155/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
index 7893ed8..f0c825f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/LogicalTupleFilter.java
@@ -170,4 +170,30 @@ public class LogicalTupleFilter extends TupleFilter implements IOptimizeableTupl
return transformer.visit(this);
}
+ @Override
+ public boolean equals(Object other) {
+ if (other == this) {
+ return true;
+ }
+ if (!(other instanceof LogicalTupleFilter)) {
+ return false;
+ }
+ final LogicalTupleFilter otherFilter = (LogicalTupleFilter) other;
+ if (otherFilter.operator != this.operator || otherFilter.children.size() != this.children.size()) {
+ return false;
+ }
+
+ for (int i = 0; i < otherFilter.children.size(); i++) {
+ if (!otherFilter.children.get(i).equals(this.children.get(i))) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return (operator == null ? 0 : operator.hashCode()) + 31 * this.children.hashCode();
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c2b69155/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 5ba8726..09b41f5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -99,6 +99,11 @@ public abstract class TupleFilter {
addChild(c); // subclass overrides addChild()
}
+ final public void addChildren(TupleFilter... children) {
+ for (TupleFilter c : children)
+ addChild(c); // subclass overrides addChild()
+ }
+
public List<? extends TupleFilter> getChildren() {
return children;
}
@@ -121,6 +126,44 @@ public abstract class TupleFilter {
}
/**
+ * The storage level dislike NOT logic
+ */
+ public TupleFilter removeNot() {
+ return removeNotInternal(this);
+ }
+
+ private TupleFilter removeNotInternal(TupleFilter filter) {
+ FilterOperatorEnum op = filter.getOperator();
+
+ if (!(filter instanceof LogicalTupleFilter)) {
+ return filter;
+ }
+
+ LogicalTupleFilter logicalFilter = (LogicalTupleFilter) filter;
+
+ switch (logicalFilter.operator) {
+ case NOT:
+ assert (filter.children.size() == 1);
+ TupleFilter reverse = filter.children.get(0).reverse();
+ return removeNotInternal(reverse);
+ case AND:
+ LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
+ for (TupleFilter child : logicalFilter.children) {
+ andFilter.addChild(removeNotInternal(child));
+ }
+ return andFilter;
+ case OR:
+ LogicalTupleFilter orFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
+ for (TupleFilter child : logicalFilter.children) {
+ orFilter.addChild(removeNotInternal(child));
+ }
+ return orFilter;
+ default:
+ throw new IllegalStateException("This filter is unexpected: " + filter);
+ }
+ }
+
+ /**
* flatten to OR-AND filter, (A AND B AND ..) OR (C AND D AND ..) OR ..
* flatten filter will ONLY contain AND and OR , no NOT will exist.
* This will help to decide scan ranges.
http://git-wip-us.apache.org/repos/asf/kylin/blob/c2b69155/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
new file mode 100644
index 0000000..e17f4d1
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.filter;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TupleFilterTest {
+ @Test
+ // true ==> true
+ public void removeNotTest0() {
+ TupleFilter constFilter = ConstantTupleFilter.TRUE;
+ Assert.assertEquals(constFilter, constFilter.removeNot());
+ }
+
+ @Test
+ // Not(true) ==> false
+ public void removeNotTest1() {
+ TupleFilter notFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.NOT);
+ notFilter.addChild(ConstantTupleFilter.TRUE);
+ Assert.assertEquals(ConstantTupleFilter.FALSE, notFilter.removeNot());
+ }
+
+ @Test
+ // Not(And(true, false)) ==> Or(false, true)
+ public void removeNotTest2() {
+ TupleFilter notFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.NOT);
+ TupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ andFilter.addChildren(ConstantTupleFilter.TRUE, ConstantTupleFilter.FALSE);
+ notFilter.addChild(andFilter);
+
+ TupleFilter orFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
+ orFilter.addChildren(ConstantTupleFilter.FALSE, ConstantTupleFilter.TRUE);
+ Assert.assertEquals(orFilter, notFilter.removeNot());
+ }
+
+ @Test
+ // And(Not(true), false) ==> And(false, false)
+ public void removeNotTest3() {
+ TupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ TupleFilter notFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.NOT);
+ notFilter.addChild(ConstantTupleFilter.TRUE);
+ andFilter.addChildren(notFilter, ConstantTupleFilter.FALSE);
+
+ TupleFilter andFilter2 = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ andFilter2.addChildren(ConstantTupleFilter.FALSE, ConstantTupleFilter.FALSE);
+ Assert.assertEquals(andFilter2, andFilter.removeNot());
+ }
+}
[40/50] [abbrv] kylin git commit: KYLIN-3204 fix potentially unclosed
resources
Posted by li...@apache.org.
KYLIN-3204 fix potentially unclosed resources
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b07a0566
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b07a0566
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b07a0566
Branch: refs/heads/sync
Commit: b07a05666d5474a9089fcd38219ffa204809ba10
Parents: 4c31b1d
Author: etherge <et...@163.com>
Authored: Tue Jan 30 17:07:27 2018 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Jan 30 17:23:32 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/source/jdbc/JdbcExplorer.java | 47 +++++++++++---------
.../org/apache/kylin/source/jdbc/SqlUtil.java | 11 ++---
2 files changed, 28 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b07a0566/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index c34597a..2dd8f9a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -18,7 +18,6 @@
package org.apache.kylin.source.jdbc;
-import java.io.IOException;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
@@ -29,7 +28,6 @@ import java.util.List;
import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.DBUtils;
import org.apache.kylin.common.util.Pair;
@@ -106,7 +104,6 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
DBUtils.closeQuietly(con);
}
-
TableExtDesc tableExtDesc = new TableExtDesc();
tableExtDesc.setIdentity(tableDesc.getIdentity());
tableExtDesc.setUuid(UUID.randomUUID().toString());
@@ -208,20 +205,23 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
return new String[] { dropView, dropTable, createSql };
}
- private void executeSQL(String sql) throws CommandNeedRetryException, IOException, SQLException {
+ private void executeSQL(String sql) throws SQLException {
Connection con = SqlUtil.getConnection(dbconf);
- logger.info(String.format(sql));
- SqlUtil.execUpdateSQL(con, sql);
- DBUtils.closeQuietly(con);
+ logger.info(String.format("Executing sql : %s"), sql);
+ try {
+ SqlUtil.execUpdateSQL(con, sql);
+ } finally {
+ DBUtils.closeQuietly(con);
+ }
}
- private void executeSQL(String[] sqls) throws CommandNeedRetryException, IOException, SQLException {
- Connection con = SqlUtil.getConnection(dbconf);
- for (String sql : sqls) {
- logger.info(String.format(sql));
- SqlUtil.execUpdateSQL(con, sql);
+ private void executeSQL(String[] sqls) throws SQLException {
+ try (Connection con = SqlUtil.getConnection(dbconf)) {
+ for (String sql : sqls) {
+ logger.info(String.format("Executing sql : %s"), sql);
+ SqlUtil.execUpdateSQL(con, sql);
+ }
}
- DBUtils.closeQuietly(con);
}
@Override
@@ -237,32 +237,35 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
KylinConfig config = KylinConfig.getInstanceFromEnv();
String tmpDatabase = config.getHiveDatabaseForIntermediateTable();
- String tmpView = tmpDatabase + ".kylin_eval_query_" + UUID.nameUUIDFromBytes(query.getBytes()).toString().replaceAll("-", "");
+ String tmpView = tmpDatabase + ".kylin_eval_query_"
+ + UUID.nameUUIDFromBytes(query.getBytes()).toString().replaceAll("-", "");
String dropViewSql = "DROP VIEW IF EXISTS " + tmpView;
String evalViewSql = "CREATE VIEW " + tmpView + " as " + query;
+ Connection con = null;
+ ResultSet rs = null;
try {
logger.debug("Removing duplicate view {}", tmpView);
executeSQL(dropViewSql);
logger.debug("Creating view {} for query: {}", tmpView, query);
executeSQL(evalViewSql);
logger.debug("Evaluating query columns' metadata");
- Connection con = SqlUtil.getConnection(dbconf);
+ con = SqlUtil.getConnection(dbconf);
DatabaseMetaData dbmd = con.getMetaData();
- ResultSet rs = dbmd.getColumns(null, tmpDatabase, tmpView, null);
+ rs = dbmd.getColumns(null, tmpDatabase, tmpView, null);
ColumnDesc[] result = extractColumnFromMeta(rs);
- DBUtils.closeQuietly(rs);
- DBUtils.closeQuietly(con);
return result;
- } catch (Exception e) {
+ } catch (SQLException e) {
throw new RuntimeException("Cannot evaluate metadata of query: " + query, e);
} finally {
+ DBUtils.closeQuietly(con);
+ DBUtils.closeQuietly(rs);
try {
- logger.debug("Cleaning up.");
+ logger.debug("Cleaning up temp view.");
executeSQL(dropViewSql);
- } catch (Exception e) {
- logger.warn("Cannot drop temp view of query: {}", query, e);
+ } catch (SQLException e) {
+ logger.warn("Failed to clean up temp view of query: {}", query, e);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b07a0566/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
index 715bb99..f86fd7e 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
@@ -20,6 +20,7 @@ package org.apache.kylin.source.jdbc;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.util.Random;
@@ -50,15 +51,9 @@ public class SqlUtil {
}
}
- public static void execUpdateSQL(Connection db, String sql) {
- Statement statement = null;
- try {
- statement = db.createStatement();
+ public static void execUpdateSQL(Connection db, String sql) throws SQLException {
+ try (Statement statement = db.createStatement()) {
statement.executeUpdate(sql);
- } catch (Exception e) {
- logger.error("", e);
- } finally {
- closeResources(null, statement);
}
}
[20/50] [abbrv] kylin git commit: Update decimal's max precision to
38, be aligned with Hive
Posted by li...@apache.org.
Update decimal's max precision to 38, be aligned with Hive
(cherry picked from commit 30e09d8c8198ed074658d2664676a5243a0690dc)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/543124da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/543124da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/543124da
Branch: refs/heads/sync
Commit: 543124dab1ec1e6f23ea8cc63c97abc89a1eae0d
Parents: 59d5064
Author: Yifan Zhang <ev...@gmail.com>
Authored: Thu Jan 18 23:44:25 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../query/calcite/KylinRelDataTypeSystem.java | 11 +++
.../calcite/KylinRelDataTypeSystemTest.java | 78 ++++++++++++++++++++
2 files changed, 89 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/543124da/query/src/main/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystem.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystem.java b/query/src/main/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystem.java
index 060747a..35b4c3d 100644
--- a/query/src/main/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystem.java
+++ b/query/src/main/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystem.java
@@ -45,4 +45,15 @@ public class KylinRelDataTypeSystem extends RelDataTypeSystemImpl {
}
return argumentType;
}
+
+ /**
+ * Hive support decimal with 38 digits, kylin should align
+ *
+ * @see org.apache.calcite.rel.type.RelDataTypeSystem
+ * @see <a href="https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-DecimalsdecimalDecimals">Hive/LanguageManualTypes-Decimals</a>
+ */
+ @Override
+ public int getMaxNumericPrecision() {
+ return 38;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/543124da/query/src/test/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystemTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystemTest.java b/query/src/test/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystemTest.java
new file mode 100644
index 0000000..e9ef548
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/calcite/KylinRelDataTypeSystemTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query.calcite;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.sql.type.BasicSqlType;
+import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.query.schema.OLAPTable;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class KylinRelDataTypeSystemTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testLegalDecimalType() {
+ RelDataTypeSystem typeSystem = new KylinRelDataTypeSystem();
+ RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(typeSystem);
+
+ DataType dataType = DataType.getType("decimal(30, 10)");
+ RelDataType relDataType = OLAPTable.createSqlType(typeFactory, dataType, true);
+
+ Assert.assertTrue(relDataType instanceof BasicSqlType);
+ Assert.assertEquals(relDataType.getSqlTypeName(), SqlTypeName.DECIMAL);
+ Assert.assertEquals(relDataType.getPrecision(), 30);
+ Assert.assertTrue(relDataType.getPrecision() <= typeSystem.getMaxNumericPrecision());
+ Assert.assertEquals(relDataType.getScale(), 10);
+ Assert.assertTrue(relDataType.getScale() <= typeSystem.getMaxNumericScale());
+ }
+
+ @Test
+ public void testIllegalDecimalType() {
+ RelDataTypeSystem typeSystem = new KylinRelDataTypeSystem();
+ RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(typeSystem);
+
+ DataType dataType = DataType.getType("decimal(40, 10)");
+ RelDataType relDataType = OLAPTable.createSqlType(typeFactory, dataType, true);
+
+ Assert.assertTrue(relDataType instanceof BasicSqlType);
+ Assert.assertEquals(relDataType.getSqlTypeName(), SqlTypeName.DECIMAL);
+ Assert.assertTrue(typeSystem.getMaxNumericPrecision() < 40);
+ Assert.assertEquals(relDataType.getPrecision(), typeSystem.getMaxNumericPrecision());
+ Assert.assertEquals(relDataType.getScale(), 10);
+ Assert.assertTrue(relDataType.getScale() <= typeSystem.getMaxNumericScale());
+ }
+}
[02/50] [abbrv] kylin git commit: KYLIN-3167,
fulfil datatype's precision
Posted by li...@apache.org.
KYLIN-3167, fulfil datatype's precision
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/55365b68
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/55365b68
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/55365b68
Branch: refs/heads/sync
Commit: 55365b68dfa021f5c14acbd79e2417f55b294473
Parents: 0988c84
Author: Cheng Wang <ch...@kyligence.io>
Authored: Tue Jan 16 14:55:36 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../kylin/source/hive/BeelineHiveClient.java | 27 +++++++++--
.../source/hive/BeelineHIveClientTest.java | 51 ++++++++++++++++++++
2 files changed, 74 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/55365b68/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index 314402c..747b1bb 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -114,7 +114,7 @@ public class BeelineHiveClient implements IHiveClient {
}
return count;
}
-
+
@Override
public void executeHQL(String hql) throws IOException {
throw new UnsupportedOperationException();
@@ -132,7 +132,11 @@ public class BeelineHiveClient implements IHiveClient {
List<HiveTableMeta.HiveTableColumnMeta> allColumns = Lists.newArrayList();
while (columns.next()) {
- allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columns.getString(4), columns.getString(6), columns.getString(12)));
+ String columnName = columns.getString(4);
+ String dataType = columns.getString(6);
+ String comment = columns.getString(12);
+ dataType = considerDataTypePrecision(dataType, columns.getString(7), columns.getString(9));
+ allColumns.add(new HiveTableMeta.HiveTableColumnMeta(columnName, dataType, comment));
}
builder.setAllColumns(allColumns);
DBUtils.closeQuietly(columns);
@@ -143,6 +147,19 @@ public class BeelineHiveClient implements IHiveClient {
return builder.createHiveTableMeta();
}
+ public static String considerDataTypePrecision(String dataType, String precision, String scale) {
+ if ("VARCHAR".equalsIgnoreCase(dataType) || "CHAR".equalsIgnoreCase(dataType)) {
+ if (null != precision)
+ dataType = new StringBuilder(dataType).append("(").append(precision).append(")").toString();
+ }
+ if ("DECIMAL".equalsIgnoreCase(dataType) || "NUMERIC".equalsIgnoreCase(dataType)) {
+ if (precision != null && scale != null)
+ dataType = new StringBuilder(dataType).append("(").append(precision).append(",").append(scale)
+ .append(")").toString();
+ }
+ return dataType;
+ }
+
private void extractHiveTableMeta(ResultSet resultSet, HiveTableMetaBuilder builder) throws SQLException {
while (resultSet.next()) {
@@ -156,7 +173,8 @@ public class BeelineHiveClient implements IHiveClient {
if ("".equals(resultSet.getString(1).trim())) {
break;
}
- partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(), resultSet.getString(2).trim(), resultSet.getString(3).trim()));
+ partitionColumns.add(new HiveTableMeta.HiveTableColumnMeta(resultSet.getString(1).trim(),
+ resultSet.getString(2).trim(), resultSet.getString(3).trim()));
}
builder.setPartitionColumns(partitionColumns);
}
@@ -213,7 +231,8 @@ public class BeelineHiveClient implements IHiveClient {
public static void main(String[] args) throws SQLException {
- BeelineHiveClient loader = new BeelineHiveClient("-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
+ BeelineHiveClient loader = new BeelineHiveClient(
+ "-n root --hiveconf hive.security.authorization.sqlstd.confwhitelist.append='mapreduce.job.*|dfs.*' -u 'jdbc:hive2://sandbox:10000'");
//BeelineHiveClient loader = new BeelineHiveClient(StringUtils.join(args, " "));
HiveTableMeta hiveTableMeta = loader.getHiveTableMeta("default", "test_kylin_fact_part");
System.out.println(hiveTableMeta);
http://git-wip-us.apache.org/repos/asf/kylin/blob/55365b68/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java
----------------------------------------------------------------------
diff --git a/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java
new file mode 100644
index 0000000..00a25e5
--- /dev/null
+++ b/source-hive/src/test/java/org/apache/kylin/source/hive/BeelineHIveClientTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.source.hive;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class BeelineHIveClientTest {
+ @Test
+ public void testBasics() {
+ String dataType = "varchar";
+ String precision = "60";
+ String scale = null;
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("varchar(60)", dataType);
+
+ dataType = "char";
+ precision = "50";
+ scale = null;
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("char(50)", dataType);
+
+ dataType = "decimal";
+ precision = "8";
+ scale = "4";
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("decimal(8,4)", dataType);
+
+ dataType = "numeric";
+ precision = "7";
+ scale = "3";
+ dataType = BeelineHiveClient.considerDataTypePrecision(dataType, precision, scale);
+ Assert.assertEquals("numeric(7,3)", dataType);
+ }
+}
[36/50] [abbrv] kylin git commit: minor, fix some typos
Posted by li...@apache.org.
minor, fix some typos
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9198cfe8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9198cfe8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9198cfe8
Branch: refs/heads/sync
Commit: 9198cfe8f1439246eff6abff38b3869453325aca
Parents: 74e3f61
Author: lidongsjtu <li...@apache.org>
Authored: Sun Jan 28 19:43:00 2018 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sun Jan 28 19:43:00 2018 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/source/hive/HiveMetadataExplorer.java | 4 ++--
.../src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9198cfe8/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index cb3eb02..9d4cc53 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -185,7 +185,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
@Override
public ColumnDesc[] evalQueryMetadata(String query) {
if (StringUtils.isEmpty(query)) {
- throw new RuntimeException("Evalutate query shall not be empty.");
+ throw new RuntimeException("Evaluate query shall not be empty.");
}
KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -204,7 +204,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
HiveTableMeta hiveTableMeta = hiveClient.getHiveTableMeta(tmpDatabase, tmpView);
return extractColumnFromMeta(hiveTableMeta);
} catch (Exception e) {
- throw new RuntimeException("Cannot evalutate metadata of query: " + query, e);
+ throw new RuntimeException("Cannot evaluate metadata of query: " + query, e);
} finally {
try {
logger.debug("Cleaning up.");
http://git-wip-us.apache.org/repos/asf/kylin/blob/9198cfe8/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index d96a68d..c34597a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -232,7 +232,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
@Override
public ColumnDesc[] evalQueryMetadata(String query) {
if (StringUtils.isEmpty(query)) {
- throw new RuntimeException("Evalutate query shall not be empty.");
+ throw new RuntimeException("Evaluate query shall not be empty.");
}
KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -256,7 +256,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
DBUtils.closeQuietly(con);
return result;
} catch (Exception e) {
- throw new RuntimeException("Cannot evalutate metadata of query: " + query, e);
+ throw new RuntimeException("Cannot evaluate metadata of query: " + query, e);
} finally {
try {
logger.debug("Cleaning up.");
[37/50] [abbrv] kylin git commit: minor, stablize
org.apache.kylin.dict.TrieDictionaryForestTest#englishWordsTest
Posted by li...@apache.org.
minor, stablize org.apache.kylin.dict.TrieDictionaryForestTest#englishWordsTest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5eedd135
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5eedd135
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5eedd135
Branch: refs/heads/sync
Commit: 5eedd135d27da96c2242079fd9e188fce0518f90
Parents: 9198cfe
Author: lidongsjtu <li...@apache.org>
Authored: Sun Jan 28 19:55:07 2018 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Sun Jan 28 19:55:07 2018 +0800
----------------------------------------------------------------------
.../kylin/dict/TrieDictionaryForestTest.java | 28 ++++++++++++++------
1 file changed, 20 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5eedd135/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index 099075c..1f63111 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -43,6 +43,7 @@ import java.util.NoSuchElementException;
import java.util.Random;
import java.util.TreeSet;
+import org.apache.kylin.common.util.Bytes;
import org.junit.Ignore;
import org.junit.Test;
@@ -187,8 +188,14 @@ public class TrieDictionaryForestTest {
str.add("party");
str.add("parties");
str.add("paint");
- String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
- + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk" + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
+ String longStr = "paintjkjdfklajkdljfkdsajklfjklsadjkjekjrklewjrklewjklrjklewjkljkljkljkljweklrjewkljrklewjrlkjewkljrkljkljkjlkjjkljkljkljkljlkjlkjlkjljdfadfads"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk"
+ + "dddddddddddddddddddddddddddddddddddddddddddddddddkfjadslkfjdsakljflksadjklfjklsjfkljwelkrjewkljrklewjklrjelkwjrklewjrlkjwkljerklkljlkjrlkwejrk";
System.out.println("The length of the long string is " + longStr.length());
str.add(longStr);
@@ -740,7 +747,8 @@ public class TrieDictionaryForestTest {
System.out.println("times:" + i);
}
- System.out.println("compare build time. Old trie : " + oldDictTotalBuildTime / 1000.0 + "s.New trie : " + newDictTotalBuildTime / 1000.0 + "s");
+ System.out.println("compare build time. Old trie : " + oldDictTotalBuildTime / 1000.0 + "s.New trie : "
+ + newDictTotalBuildTime / 1000.0 + "s");
}
private void evaluateDataSize(ArrayList<String> list) {
@@ -806,7 +814,8 @@ public class TrieDictionaryForestTest {
benchmark("Benchmark", dict, set, map, strArray, array);
}
- private static int benchmark(String msg, TrieDictionaryForest<String> dict, TreeSet<String> set, HashMap<String, Integer> map, String[] strArray, byte[][] array) {
+ private static int benchmark(String msg, TrieDictionaryForest<String> dict, TreeSet<String> set,
+ HashMap<String, Integer> map, String[] strArray, byte[][] array) {
int n = set.size();
int times = Math.max(10 * 1000 * 1000 / n, 1); // run 10 million lookups
int keep = 0; // make sure JIT don't OPT OUT function calls under test
@@ -889,7 +898,7 @@ public class TrieDictionaryForestTest {
assertEquals(id, dict.getIdFromValue(value));
assertEquals(value, dict.getValueFromId(id));
- assertArrayEquals(value.getBytes(), dict.getValueByteFromId(id));
+ assertArrayEquals(Bytes.toBytes(value), dict.getValueByteFromId(id));
}
//test not found value
@@ -932,14 +941,16 @@ public class TrieDictionaryForestTest {
}
public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId) {
- TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId);
+ TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(),
+ baseId);
for (String s : strs)
b.addValue(s);
return b;
}
public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId, int treeSize) {
- TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId);
+ TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(),
+ baseId);
b.setMaxTrieTreeSize(treeSize);
for (String s : strs) {
b.addValue(s);
@@ -948,7 +959,8 @@ public class TrieDictionaryForestTest {
}
public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterator<String> strs, int baseId, int treeSize) {
- TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId);
+ TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(),
+ baseId);
b.setMaxTrieTreeSize(treeSize);
while (strs.hasNext())
b.addValue(strs.next());