You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/07/25 09:44:48 UTC
[01/24] kylin git commit: KYLIN-2671 fix cache bug
Repository: kylin
Updated Branches:
refs/heads/2.1.x aee88fe94 -> e2bd2eb75
KYLIN-2671 fix cache bug
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8b236025
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8b236025
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8b236025
Branch: refs/heads/2.1.x
Commit: 8b2360254e0af83b2ca00a01d8fc61d91ade0779
Parents: 4496135
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Jul 4 15:08:19 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Jul 4 15:21:40 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller/QueryController.java | 7 ++++---
.../apache/kylin/rest/controller2/QueryControllerV2.java | 10 ++++++----
2 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b236025/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 61250ea..b44f66f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -82,9 +82,10 @@ public class QueryController extends BasicController {
@RequestMapping(value = "/query/prestate", method = RequestMethod.POST, produces = "application/json")
@ResponseBody
public SQLResponse prepareQuery(@RequestBody PrepareSqlRequest sqlRequest) {
- Map<String, String> toggles = Maps.newHashMap();
- toggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
- BackdoorToggles.addToggles(toggles);
+ Map<String, String> newToggles = Maps.newHashMap();
+ newToggles.putAll(sqlRequest.getBackdoorToggles());
+ newToggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
+ sqlRequest.setBackdoorToggles(newToggles);
return queryService.doQueryWithCache(sqlRequest);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8b236025/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
index a1b65a0..9343d4f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
@@ -27,7 +27,6 @@ import java.util.Map;
import javax.servlet.http.HttpServletResponse;
-import com.google.common.collect.Maps;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
@@ -58,6 +57,8 @@ import org.supercsv.io.CsvListWriter;
import org.supercsv.io.ICsvListWriter;
import org.supercsv.prefs.CsvPreference;
+import com.google.common.collect.Maps;
+
/**
* Handle query requests.
*
@@ -87,9 +88,10 @@ public class QueryControllerV2 extends BasicController {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
public EnvelopeResponse prepareQueryV2(@RequestBody PrepareSqlRequest sqlRequest) {
- Map<String, String> toggles = Maps.newHashMap();
- toggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
- BackdoorToggles.addToggles(toggles);
+ Map<String, String> newToggles = Maps.newHashMap();
+ newToggles.putAll(sqlRequest.getBackdoorToggles());
+ newToggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
+ sqlRequest.setBackdoorToggles(newToggles);
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, queryService.doQueryWithCache(sqlRequest), "");
}
[14/24] kylin git commit: update kylin-it engine
Posted by li...@apache.org.
update kylin-it engine
KryoRegistrator add ComputedColumnDesc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1b177a59
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1b177a59
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1b177a59
Branch: refs/heads/2.1.x
Commit: 1b177a591b35f5a708b5df00fc589d5d05e1d78d
Parents: a2cb533
Author: 许益铭 <xu...@hys-inc.cn>
Authored: Thu Jul 6 18:10:01 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jul 21 16:13:02 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/metadata/model/ComputedColumnDesc.java | 4 +++-
.../java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java | 1 +
.../test_case_data/localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b177a59/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index 540b5fc..4a50ba7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -22,8 +22,10 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import org.apache.kylin.metadata.model.tool.CalciteParser;
+import java.io.Serializable;
+
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class ComputedColumnDesc {
+public class ComputedColumnDesc implements Serializable{
@JsonProperty
private String tableIdentity;
@JsonProperty
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b177a59/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index 106653f..1980343 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -266,6 +266,7 @@ public class KylinKryoRegistrator implements KryoRegistrator {
kyroClasses.add(org.apache.kylin.metadata.datatype.StringSerializer.class);
kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc.class);
kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.class);
+ kyroClasses.add(org.apache.kylin.metadata.model.ComputedColumnDesc.class);
kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class);
kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class);
kyroClasses.add(org.apache.kylin.metadata.model.DatabaseDesc.class);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b177a59/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index 28a63d5..27acdd3 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -610,7 +610,7 @@
"status_need_notify": [],
"auto_merge_time_ranges": null,
"retention_range": 0,
- "engine_type": 2,
+ "engine_type": 4,
"storage_type": 2,
"override_kylin_properties": {
"kylin.cube.algorithm": "LAYER"
[18/24] kylin git commit: #1671 export as csv API from GET to POST
Posted by li...@apache.org.
#1671 export as csv API from GET to POST
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5cb6abaf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5cb6abaf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5cb6abaf
Branch: refs/heads/2.1.x
Commit: 5cb6abaf9331d737fa8274c7ee0a6288b1b7ec15
Parents: 8898213
Author: Roger Shi <ro...@hotmail.com>
Authored: Mon Jul 24 18:41:22 2017 +0800
Committer: Roger Shi <ro...@hotmail.com>
Committed: Mon Jul 24 18:56:07 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller2/QueryControllerV2.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cb6abaf/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
index 2ef4415..7a74765 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
@@ -152,10 +152,10 @@ public class QueryControllerV2 extends BasicController {
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, data, "");
}
- @RequestMapping(value = "/query/format/{format}", method = RequestMethod.GET, produces = {
+ @RequestMapping(value = "/query/format/{format}", method = RequestMethod.POST, produces = {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
- public void downloadQueryResultV2(@PathVariable String format, SQLRequest sqlRequest,
+ public void downloadQueryResultV2(@PathVariable String format, @RequestBody SQLRequest sqlRequest,
HttpServletResponse response) {
SQLResponse result = queryService.doQueryWithCache(sqlRequest);
@@ -169,7 +169,7 @@ public class QueryControllerV2 extends BasicController {
List<String> headerList = new ArrayList<String>();
for (SelectedColumnMeta column : result.getColumnMetas()) {
- headerList.add(column.getName());
+ headerList.add(column.getLabel());
}
String[] headers = new String[headerList.size()];
[15/24] kylin git commit: 1653 bugfix:remove the usage that delete
resource first and then put.Use put without check (#1664)
Posted by li...@apache.org.
1653 bugfix:remove the usage that delete resource first and then put.Use put without check (#1664)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c53ef3f0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c53ef3f0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c53ef3f0
Branch: refs/heads/2.1.x
Commit: c53ef3f0a71ba085fb84917a55ded0f122cf9860
Parents: cfaeb7e
Author: Jiatao Tao <24...@qq.com>
Authored: Sun Jul 23 22:17:03 2017 -0500
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Jul 24 12:47:39 2017 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/ResourceStore.java | 17 +++++++++++++++++
.../org/apache/kylin/rest/service/AclService.java | 3 +--
.../kylin/rest/service/AclTableMigrationTool.java | 8 ++++----
.../apache/kylin/rest/service/QueryService.java | 8 ++++----
.../org/apache/kylin/rest/service/UserService.java | 3 +--
5 files changed, 27 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c53ef3f0/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 997424b..ac29dc0 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -223,6 +223,23 @@ abstract public class ResourceStore {
/**
* overwrite a resource without write conflict check
*/
+ final public <T extends RootPersistentEntity> void putResourceWithoutCheck(String resPath, T obj, long ts,
+ Serializer<T> serializer) throws IOException {
+ resPath = norm(resPath);
+ logger.trace("Directly saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
+ ByteArrayOutputStream buf = new ByteArrayOutputStream();
+ DataOutputStream dout = new DataOutputStream(buf);
+ serializer.serialize(obj, dout);
+ dout.close();
+ buf.close();
+ ByteArrayInputStream is = new ByteArrayInputStream(buf.toByteArray());
+ putResourceCheckpoint(resPath, is, ts);
+ is.close();
+ }
+
+ /**
+ * overwrite a resource without write conflict check
+ */
final public void putResource(String resPath, InputStream content, long ts) throws IOException {
resPath = norm(resPath);
logger.trace("Directly saving resource " + resPath + " (Store " + kylinConfig.getMetadataUrl() + ")");
http://git-wip-us.apache.org/repos/asf/kylin/blob/c53ef3f0/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index 79ed4aa..ac5d098 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -232,7 +232,6 @@ public class AclService implements MutableAclService {
try {
String id = getQueryKeyById(String.valueOf(mutableAcl.getObjectIdentity().getIdentifier()));
AclRecord record = aclStore.getResource(id, AclRecord.class, SERIALIZER);
- aclStore.deleteResource(id);
if (mutableAcl.getParentAcl() != null) {
record.setParentDomainObjectInfo(new DomainObjectInfo(mutableAcl.getParentAcl().getObjectIdentity()));
}
@@ -252,7 +251,7 @@ public class AclService implements MutableAclService {
AceInfo aceInfo = new AceInfo(ace);
allAceInfo.put(String.valueOf(aceInfo.getSidInfo().getSid()), aceInfo);
}
- aclStore.putResource(id, record, 0, SERIALIZER);
+ aclStore.putResourceWithoutCheck(id, record, System.currentTimeMillis(), SERIALIZER);
logger.debug("ACL of " + mutableAcl.getObjectIdentity() + " updated successfully.");
} catch (IOException e) {
throw new InternalErrorException(e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c53ef3f0/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
index bb07c22..fe91f52 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
@@ -130,8 +130,8 @@ public class AclTableMigrationTool {
record.setOwnerInfo(getOwnerSidInfo(result));
record.setEntriesInheriting(getInheriting(result));
record.setAllAceInfo(getAllAceInfo(result));
- store.deleteResource(AclService.getQueryKeyById(object.getId()));
- store.putResource(AclService.getQueryKeyById(object.getId()), record, 0, AclService.SERIALIZER);
+ store.putResourceWithoutCheck(AclService.getQueryKeyById(object.getId()), record,
+ System.currentTimeMillis(), AclService.SERIALIZER);
result = rs.next();
}
}
@@ -148,8 +148,8 @@ public class AclTableMigrationTool {
Result result = rs.next();
while (result != null) {
ManagedUser user = hbaseRowToUser(result);
- store.deleteResource(UserService.getId(user.getUsername()));
- store.putResource(UserService.getId(user.getUsername()), user, 0, UserService.SERIALIZER);
+ store.putResourceWithoutCheck(UserService.getId(user.getUsername()), user,
+ System.currentTimeMillis(), UserService.SERIALIZER);
result = rs.next();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c53ef3f0/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index f3402ef..4061c82 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -186,8 +186,8 @@ public class QueryService extends BasicService {
queries.add(query);
Query[] queryArray = new Query[queries.size()];
QueryRecord record = new QueryRecord(queries.toArray(queryArray));
- queryStore.deleteResource(getQueryKeyById(creator));
- queryStore.putResource(getQueryKeyById(creator), record, 0, QueryRecordSerializer.getInstance());
+ queryStore.putResourceWithoutCheck(getQueryKeyById(creator), record, System.currentTimeMillis(),
+ QueryRecordSerializer.getInstance());
return;
}
@@ -210,8 +210,8 @@ public class QueryService extends BasicService {
}
Query[] queryArray = new Query[queries.size()];
QueryRecord record = new QueryRecord(queries.toArray(queryArray));
- queryStore.deleteResource(getQueryKeyById(creator));
- queryStore.putResource(getQueryKeyById(creator), record, 0, QueryRecordSerializer.getInstance());
+ queryStore.putResourceWithoutCheck(getQueryKeyById(creator), record, System.currentTimeMillis(),
+ QueryRecordSerializer.getInstance());
return;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c53ef3f0/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 6682c03..16fafa3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -82,9 +82,8 @@ public class UserService implements UserDetailsManager {
Preconditions.checkState(user instanceof ManagedUser, "User {} is not ManagedUser", user);
ManagedUser managedUser = (ManagedUser) user;
try {
- deleteUser(user.getUsername());
String id = getId(user.getUsername());
- aclStore.putResource(id, managedUser, 0, SERIALIZER);
+ aclStore.putResourceWithoutCheck(id, managedUser, System.currentTimeMillis(), SERIALIZER);
logger.trace("update user : {}", user.getUsername());
} catch (IOException e) {
throw new InternalErrorException(e);
[20/24] kylin git commit: minor, avoid NPE in doPushDownQuery check
Posted by li...@apache.org.
minor, avoid NPE in doPushDownQuery check
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9141bffc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9141bffc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9141bffc
Branch: refs/heads/2.1.x
Commit: 9141bffc3e2732625e8f8db8d9edaa269cc34e94
Parents: 6214ab9
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Jul 25 11:50:35 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Jul 25 11:55:10 2017 +0800
----------------------------------------------------------------------
.../test/java/org/apache/kylin/query/KylinTestBase.java | 6 +++++-
.../org/apache/kylin/rest/service/QueryService.java | 6 +++++-
.../java/org/apache/kylin/rest/util/PushDownUtil.java | 12 ++++++++----
3 files changed, 18 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9141bffc/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 3634100..e1d4a44 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -263,7 +263,11 @@ public class KylinTestBase {
} catch (SQLException sqlException) {
List<List<String>> results = Lists.newArrayList();
List<SelectedColumnMeta> columnMetas = Lists.newArrayList();
- PushDownUtil.doPushDownQuery(ProjectInstance.DEFAULT_PROJECT_NAME, sql, results, columnMetas, sqlException);
+ boolean b = PushDownUtil.doPushDownQuery(ProjectInstance.DEFAULT_PROJECT_NAME, sql, results, columnMetas,
+ sqlException);
+ if (!b) {
+ throw sqlException;
+ }
return results.size();
} finally {
if (resultSet != null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/9141bffc/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 75c16bd..f4ae06c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -487,7 +487,8 @@ public class QueryService extends BasicService {
return fakeResponse;
}
- String correctedSql = QueryUtil.massageSql(sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(), sqlRequest.getOffset());
+ String correctedSql = QueryUtil.massageSql(sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(),
+ sqlRequest.getOffset());
if (!correctedSql.equals(sqlRequest.getSql())) {
logger.info("The corrected query: " + correctedSql);
@@ -792,6 +793,9 @@ public class QueryService extends BasicService {
} catch (SQLException sqlException) {
isPushDown = PushDownUtil.doPushDownQuery(sqlRequest.getProject(), correctedSql, results, columnMetas,
sqlException);
+ if (!isPushDown) {
+ throw sqlException;
+ }
} finally {
close(resultSet, stat, conn);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9141bffc/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
index 5d7f47a..0802718 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
@@ -49,11 +49,15 @@ public class PushDownUtil {
public static boolean doPushDownQuery(String project, String sql, List<List<String>> results,
List<SelectedColumnMeta> columnMetas, SQLException sqlException) throws Exception {
- boolean isExpectedCause = (ExceptionUtils.getRootCause(sqlException).getClass()
- .equals(NoRealizationFoundException.class));
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ if (!kylinConfig.isPushDownEnabled()) {
+ return false;
+ }
+
+ Throwable rootCause = ExceptionUtils.getRootCause(sqlException);
+ boolean isExpectedCause = rootCause != null && (rootCause.getClass().equals(NoRealizationFoundException.class));
- if (isExpectedCause && kylinConfig.isPushDownEnabled()) {
+ if (isExpectedCause) {
logger.info("Query failed to utilize pre-calculation, routing to other engines", sqlException);
IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(kylinConfig.getPushDownRunnerClassName());
@@ -78,7 +82,7 @@ public class PushDownUtil {
return true;
} else {
- throw sqlException;
+ return false;
}
}
[06/24] kylin git commit: KYLIN-2743 Potential corrupt TableDesc when
loading an existing Hive table
Posted by li...@apache.org.
KYLIN-2743 Potential corrupt TableDesc when loading an existing Hive table
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fdf13892
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fdf13892
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fdf13892
Branch: refs/heads/2.1.x
Commit: fdf13892b4177a0157a5cf1621db0e3a662bfd30
Parents: adcd0b6
Author: Yang Li <li...@apache.org>
Authored: Sun Jul 16 10:12:38 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sun Jul 16 12:36:22 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/metadata/model/TableDesc.java | 9 ++++++++-
.../apache/kylin/source/hive/HiveMetadataExplorer.java | 11 ++++++++++-
2 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdf13892/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index b388f11..530732a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -61,16 +61,23 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
}
public TableDesc(TableDesc other) {
+ this.uuid = other.uuid;
+ this.lastModified = other.lastModified;
+
this.name = other.name;
this.sourceType = other.sourceType;
- this.database.setName(other.getDatabase());
this.tableType = other.tableType;
this.dataGen = other.dataGen;
+
this.columns = new ColumnDesc[other.columns.length];
for (int i = 0; i < other.columns.length; i++) {
this.columns[i] = new ColumnDesc(other.columns[i]);
this.columns[i].init(this);
}
+
+ this.project = other.project;
+ this.database.setName(other.getDatabase());
+ this.identity = other.identity;
}
public TableDesc appendColumns(ColumnDesc[] computedColumns) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/fdf13892/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index 0db4b40..00a1c3b 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -59,13 +59,18 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
}
TableDesc tableDesc = metaMgr.getTableDesc(database + "." + tableName);
+
+ // make a new TableDesc instance, don't modify the one in use
if (tableDesc == null) {
tableDesc = new TableDesc();
tableDesc.setDatabase(database.toUpperCase());
tableDesc.setName(tableName.toUpperCase());
tableDesc.setUuid(UUID.randomUUID().toString());
tableDesc.setLastModified(0);
+ } else {
+ tableDesc = new TableDesc(tableDesc);
}
+
if (hiveTableMeta.tableType != null) {
tableDesc.setTableType(hiveTableMeta.tableType);
}
@@ -95,7 +100,11 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
partitionColumnString.append(hiveTableMeta.partitionColumns.get(i).name.toUpperCase());
}
- TableExtDesc tableExtDesc = metaMgr.getTableExt(tableDesc.getIdentity());
+ TableExtDesc tableExtDesc = new TableExtDesc();
+ tableExtDesc.setName(tableDesc.getIdentity());
+ tableExtDesc.setUuid(UUID.randomUUID().toString());
+ tableExtDesc.setLastModified(0);
+
tableExtDesc.addDataSourceProp("location", hiveTableMeta.sdLocation);
tableExtDesc.addDataSourceProp("owner", hiveTableMeta.owner);
tableExtDesc.addDataSourceProp("last_access_time", String.valueOf(hiveTableMeta.lastAccessTime));
[11/24] kylin git commit: bugfix#1553 Computed column 隐式转换两端括号被截断
Posted by li...@apache.org.
bugfix#1553 Computed column 隐式转换两端括号被截断
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a2cb5335
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a2cb5335
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a2cb5335
Branch: refs/heads/2.1.x
Commit: a2cb53358e05af9093f9762157f84686569460fc
Parents: 11aa9ff
Author: Aron.tao <24...@qq.com>
Authored: Thu Jul 13 10:42:28 2017 +0800
Committer: Jiatao Tao <ji...@Kyligences-MacBook-Pro.local>
Committed: Fri Jul 21 14:11:16 2017 +0800
----------------------------------------------------------------------
.../metadata/model/tool/CalciteParser.java | 42 ++++++++++++++++++--
.../kylin/model/tool/CalciteParserTest.java | 19 ++++++++-
.../query/util/ConvertToComputedColumn.java | 3 +-
.../query/util/ConvertToComputedColumnTest.java | 11 +++++
4 files changed, 68 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2cb5335/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
index 9b80969..5352708 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/tool/CalciteParser.java
@@ -24,7 +24,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Set;
-import com.google.common.collect.Lists;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlLiteral;
@@ -39,6 +38,7 @@ import org.apache.calcite.sql.util.SqlVisitor;
import org.apache.commons.lang3.tuple.Pair;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
public class CalciteParser {
@@ -188,7 +188,11 @@ public class CalciteParser {
return sql.substring(prefix.length(), sql.length() - suffix.length());
}
- public static Pair<Integer, Integer> getReplacePos(SqlNode node, String... lines) {
+ public static Pair<Integer, Integer> getReplacePos(SqlNode node, String inputSql) {
+ if (inputSql == null) {
+ return Pair.of(0, 0);
+ }
+ String[] lines = inputSql.split("\n");
SqlParserPos pos = node.getParserPosition();
int lineStart = pos.getLineNum();
int lineEnd = pos.getEndLineNum();
@@ -201,6 +205,38 @@ public class CalciteParser {
for (int i = 0; i < lineEnd - 1; i++) {
columnEnd += lines[i].length() + 1;
}
- return Pair.of(columnStart, columnEnd);
+ //for calcite's bug CALCITE-1875
+ Pair<Integer, Integer> startEndPos = getPosWithBracketsCompletion(inputSql, columnStart, columnEnd);
+ return startEndPos;
+ }
+
+ private static Pair<Integer, Integer> getPosWithBracketsCompletion(String inputSql, int left, int right) {
+ int leftBracketNum = 0;
+ int rightBracketNum = 0;
+ String substring = inputSql.substring(left, right);
+ for (int i = 0; i < substring.length(); i++) {
+ char temp = substring.charAt(i);
+ if (temp == '(') {
+ leftBracketNum++;
+ }
+ if (temp == ')') {
+ rightBracketNum++;
+ if (leftBracketNum < rightBracketNum) {
+ while ('(' != inputSql.charAt(left - 1)) {
+ left--;
+ }
+ left--;
+ leftBracketNum++;
+ }
+ }
+ }
+ while (rightBracketNum < leftBracketNum) {
+ while (')' != inputSql.charAt(right)) {
+ right++;
+ }
+ right++;
+ rightBracketNum++;
+ }
+ return Pair.of(left, right);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2cb5335/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
index 81c1c97..2da6c9e 100644
--- a/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/model/tool/CalciteParserTest.java
@@ -95,8 +95,7 @@ public class CalciteParserTest {
for (String sql : sqls) {
SqlNode parse = ((SqlSelect) CalciteParser.parse(sql)).getSelectList().get(0);
- String[] lines = sql.split("\n");
- Pair<Integer, Integer> replacePos = CalciteParser.getReplacePos(parse, lines);
+ Pair<Integer, Integer> replacePos = CalciteParser.getReplacePos(parse, sql);
String substring = sql.substring(replacePos.getLeft(), replacePos.getRight());
Preconditions.checkArgument(substring.startsWith("a"));
Preconditions.checkArgument(substring.endsWith("b"));
@@ -120,4 +119,20 @@ public class CalciteParserTest {
assertEquals(true, CalciteParser.isNodeEqual(sn0, sn2));
assertEquals(false, CalciteParser.isNodeEqual(sn0, sn3));
}
+
+ @Test
+ public void testPosWithBrackets() throws SqlParseException {
+ String[] sqls = new String[] {
+ "select ( a + b) * (c+ d ) from t", "select (a+b) * (c+d) from t",
+ "select (a + b) * (c+ d) from t", "select (a+b) * (c+d) from t",
+ };
+
+ for (String sql : sqls) {
+ SqlNode parse = ((SqlSelect) CalciteParser.parse(sql)).getSelectList().get(0);
+ Pair<Integer, Integer> replacePos = CalciteParser.getReplacePos(parse, sql);
+ String substring = sql.substring(replacePos.getLeft(), replacePos.getRight());
+ Preconditions.checkArgument(substring.startsWith("("));
+ Preconditions.checkArgument(substring.endsWith(")"));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2cb5335/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
index c9feaab..ad42b96 100644
--- a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
+++ b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
@@ -74,7 +74,6 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
return inputSql;
}
String result = inputSql;
- String[] lines = inputSql.split("\n");
List<Pair<String, String>> toBeReplacedExp = new ArrayList<>(); //{"alias":"expression"}, like {"t1":"t1.a+t1.b+t1.c"}
for (String ccExp : computedColumn.keySet()) {
@@ -86,7 +85,7 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
return inputSql;
}
for (SqlNode node : matchedNodes) {
- Pair<Integer, Integer> startEndPos = CalciteParser.getReplacePos(node, lines);
+ Pair<Integer, Integer> startEndPos = CalciteParser.getReplacePos(node, inputSql);
int start = startEndPos.getLeft();
int end = startEndPos.getRight();
//add table alias like t1.column,if exists alias
http://git-wip-us.apache.org/repos/asf/kylin/blob/a2cb5335/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java b/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
index f9b8d8b..d9910fe 100644
--- a/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
@@ -110,4 +110,15 @@ public class ConvertToComputedColumnTest {
}
+ @Test
+ public void testCCWithBrackets() {
+ String sql0 = "select ( a + b) + (c+d \t\n) from t";
+ String expr1 = "a + b + (c + d)";
+
+ Map<String, String> map = new HashMap<>();
+ map.put("cc0", expr1);
+ ImmutableSortedMap<String, String> computedColumns = ConvertToComputedColumn.getMapSortedByValue(map);
+ Assert.assertEquals("select cc0 from t",
+ ConvertToComputedColumn.replaceComputedColumn(sql0, computedColumns));
+ }
}
[05/24] kylin git commit: Do empty check for override Kylin props
when save project.
Posted by li...@apache.org.
Do empty check for override Kylin props when save project.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/adcd0b67
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/adcd0b67
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/adcd0b67
Branch: refs/heads/2.1.x
Commit: adcd0b672958df770913f49d8967bc2c0d7a4a0b
Parents: 8307ff4
Author: nichunen <ch...@kyligence.io>
Authored: Fri Jul 14 14:08:25 2017 +0800
Committer: nichunen <ch...@kyligence.io>
Committed: Fri Jul 14 14:08:25 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/project/ProjectManager.java | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/adcd0b67/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index b801d8c..81e7647 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.project;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -28,7 +29,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -260,6 +261,20 @@ public class ProjectManager {
private void updateProject(ProjectInstance prj) throws IOException {
synchronized (prj) {
+ LinkedHashMap<String, String> overrideProps = prj.getOverrideKylinProps();
+
+ if (overrideProps != null) {
+ Iterator<Map.Entry<String, String>> iterator = overrideProps.entrySet().iterator();
+
+ while (iterator.hasNext()) {
+ Map.Entry<String, String> entry = iterator.next();
+
+ if (StringUtils.isAnyBlank(entry.getKey(), entry.getValue())) {
+ throw new IllegalStateException("Property key/value must not be blank");
+ }
+ }
+ }
+
getStore().putResource(prj.getResourcePath(), prj, PROJECT_SERIALIZER);
projectMap.put(norm(prj.getName()), prj); // triggers update broadcast
clearL2Cache();
[23/24] kylin git commit: minor,
fix bug "pick uncapable Cube when selecting lookup table"
Posted by li...@apache.org.
minor, fix bug "pick uncapable Cube when selecting lookup table"
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d188ee61
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d188ee61
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d188ee61
Branch: refs/heads/2.1.x
Commit: d188ee61c7b2caddd01201abff9678f342ad7cb3
Parents: f8e8d43
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Jul 25 16:33:26 2017 +0800
Committer: Roger Shi <ro...@hotmail.com>
Committed: Tue Jul 25 16:46:12 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d188ee61/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
index 20cb0a9..61516d2 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeCapabilityChecker.java
@@ -97,6 +97,7 @@ public class CubeCapabilityChecker {
}
} else {
logger.info("cube {} does not touch lookup table {} at all", cube.getName(), digest.factTable);
+ return result;
}
}
[02/24] kylin git commit: KYLIN-2704 Alter StorageCleanupJob to
handle no file on HDFS
Posted by li...@apache.org.
KYLIN-2704 Alter StorageCleanupJob to handle no file on HDFS
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/21d25b90
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/21d25b90
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/21d25b90
Branch: refs/heads/2.1.x
Commit: 21d25b907b76492fdb48e0050c35c733017333fc
Parents: 8b23602
Author: auphyroc99 <45...@qq.com>
Authored: Mon Jul 3 16:54:55 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Tue Jul 4 19:21:12 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/job/StorageCleanupJob.java | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/21d25b90/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
index 9b72788..8bb06b5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -18,6 +18,7 @@
package org.apache.kylin.rest.job;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
@@ -119,14 +120,18 @@ public class StorageCleanupJob extends AbstractApplication {
// GlobFilter(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()
// + "/kylin-.*");
// TODO: when first use, /kylin/kylin_metadata does not exist.
- FileStatus[] fStatus = fs.listStatus(new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()));
- for (FileStatus status : fStatus) {
- String path = status.getPath().getName();
- // System.out.println(path);
- if (path.startsWith("kylin-")) {
- String kylinJobPath = engineConfig.getHdfsWorkingDirectory() + path;
- allHdfsPathsNeedToBeDeleted.add(kylinJobPath);
+ try {
+ FileStatus[] fStatus = fs.listStatus(new Path(KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()));
+ for (FileStatus status : fStatus) {
+ String path = status.getPath().getName();
+ // System.out.println(path);
+ if (path.startsWith("kylin-")) {
+ String kylinJobPath = engineConfig.getHdfsWorkingDirectory() + path;
+ allHdfsPathsNeedToBeDeleted.add(kylinJobPath);
+ }
}
+ } catch (FileNotFoundException e) {
+ logger.info("Working Directory does not exist on HDFS. ");
}
List<String> allJobs = executableManager.getAllJobIds();
[12/24] kylin git commit: KYLIN-2633 Upgrade Spark to 2.x
Posted by li...@apache.org.
KYLIN-2633 Upgrade Spark to 2.x
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cfaeb7e7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cfaeb7e7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cfaeb7e7
Branch: refs/heads/2.1.x
Commit: cfaeb7e7eb12eefa7c0ecd40ceabeea6b5dbf62e
Parents: 309fac0
Author: 许益铭 <xu...@hys-inc.cn>
Authored: Fri Jul 7 00:13:24 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jul 21 16:13:02 2017 +0800
----------------------------------------------------------------------
assembly/pom.xml | 2 +-
build/script/download-spark.sh | 4 ++--
.../main/resources/kylin-defaults.properties | 3 ++-
.../gtrecord/GTCubeStorageQueryBase.java | 2 +-
engine-spark/pom.xml | 6 ++---
.../apache/kylin/engine/spark/SparkCubing.java | 22 +++++++++---------
.../kylin/engine/spark/SparkCubingByLayer.java | 11 +++++----
.../kylin/engine/spark/SparkHiveDemo.java | 5 ++--
.../test_case_data/sandbox/kylin.properties | 4 ++--
kylin-it/pom.xml | 24 +++++++++++++++++++-
pom.xml | 10 ++++----
server/pom.xml | 2 +-
source-kafka/pom.xml | 2 +-
13 files changed, 61 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 0a64dde..b988445 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -152,7 +152,7 @@
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
+ <artifactId>kafka_2.11</artifactId>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/build/script/download-spark.sh
----------------------------------------------------------------------
diff --git a/build/script/download-spark.sh b/build/script/download-spark.sh
index b73331e..8025591 100755
--- a/build/script/download-spark.sh
+++ b/build/script/download-spark.sh
@@ -27,8 +27,8 @@ if [[ `uname -a` =~ "Darwin" ]]; then
alias md5cmd="md5 -q"
fi
-spark_version="1.6.3"
-spark_pkg_md5="ce8a2e7529aac0f0175194061769dbd4"
+spark_version="2.1.1"
+spark_pkg_md5="195daab700e4332fcdaf7c66236de542"
if [ ! -f "build/spark-${spark_version}-bin-hadoop2.6.tgz" ]
then
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --git a/core-common/src/main/resources/kylin-defaults.properties b/core-common/src/main/resources/kylin-defaults.properties
index cb511e7..ee25637 100644
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@ -218,7 +218,7 @@ kylin.engine.spark.max-partition=5000
# Spark conf (default is in spark/conf/spark-defaults.conf)
kylin.engine.spark-conf.spark.master=yarn
-kylin.engine.spark-conf.spark.submit.deployMode=cluster
+#kylin.engine.spark-conf.spark.submit.deployMode=cluster
kylin.engine.spark-conf.spark.yarn.queue=default
kylin.engine.spark-conf.spark.executor.memory=1G
kylin.engine.spark-conf.spark.executor.cores=2
@@ -226,6 +226,7 @@ kylin.engine.spark-conf.spark.executor.instances=1
kylin.engine.spark-conf.spark.eventLog.enabled=true
kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history
kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs\:///kylin/spark-history
+kylin.engine.spark-conf.spark.hadoop.yarn.timeline-service.enabled=false
# manually upload spark-assembly jar to HDFS and then set this property will avoid repeatedly uploading jar at runtime
#kylin.engine.spark-conf.spark.yarn.jar=hdfs://namenode:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 10f735e..22f5fc9 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -83,7 +83,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
continue;
- }
+ }
scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), request.getGroups(), request.getMetrics(), request.getFilter(), request.getHavingFilter(), request.getContext());
if (!scanner.isSegmentSkipped())
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index 93b6f9b..26a6e31 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -51,19 +51,19 @@
<!-- Spark dependency -->
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.10</artifactId>
+ <artifactId>spark-core_2.11</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
+ <artifactId>spark-sql_2.11</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.10</artifactId>
+ <artifactId>spark-hive_2.11</artifactId>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 2a0981a..57fd315 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -98,7 +98,7 @@ import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.spark.api.java.function.PairFunction;
-import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.hive.HiveContext;
import org.slf4j.Logger;
@@ -179,7 +179,7 @@ public class SparkCubing extends AbstractApplication {
}
}
- private void writeDictionary(DataFrame intermediateTable, String cubeName, String segmentId) throws Exception {
+ private void writeDictionary(Dataset<Row> intermediateTable, String cubeName, String segmentId) throws Exception {
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
final CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
final CubeInstance cubeInstance = cubeManager.reloadCubeLocal(cubeName);
@@ -204,9 +204,9 @@ public class SparkCubing extends AbstractApplication {
for (Map.Entry<Integer, TblColRef> entry : tblColRefMap.entrySet()) {
final String column = columns[entry.getKey()];
final TblColRef tblColRef = entry.getValue();
- final DataFrame frame = intermediateTable.select(column).distinct();
+ final Dataset<Row> frame = intermediateTable.select(column).distinct();
- final Row[] rows = frame.collect();
+ final List<Row> rows = frame.collectAsList();
dictionaryMap.put(tblColRef, DictionaryGenerator.buildDictionary(tblColRef.getType(), new IterableDictionaryValueEnumerator(new Iterable<String>() {
@Override
public Iterator<String> iterator() {
@@ -215,13 +215,13 @@ public class SparkCubing extends AbstractApplication {
@Override
public boolean hasNext() {
- return i < rows.length;
+ return i < rows.size();
}
@Override
public String next() {
if (hasNext()) {
- final Row row = rows[i++];
+ final Row row = rows.get(i++);
final Object o = row.get(0);
return o != null ? o.toString() : null;
} else {
@@ -367,7 +367,7 @@ public class SparkCubing extends AbstractApplication {
final JavaPairRDD<byte[], byte[]> javaPairRDD = javaRDD.glom().mapPartitionsToPair(new PairFlatMapFunction<Iterator<List<List<String>>>, byte[], byte[]>() {
@Override
- public Iterable<Tuple2<byte[], byte[]>> call(Iterator<List<List<String>>> listIterator) throws Exception {
+ public Iterator<Tuple2<byte[], byte[]>> call(Iterator<List<List<String>>> listIterator) throws Exception {
long t = System.currentTimeMillis();
prepare();
@@ -390,7 +390,7 @@ public class SparkCubing extends AbstractApplication {
throw new RuntimeException(e);
}
System.out.println("build partition cost: " + (System.currentTimeMillis() - t) + "ms");
- return sparkCuboidWriter.getResult();
+ return sparkCuboidWriter.getResult().iterator();
}
});
@@ -430,7 +430,7 @@ public class SparkCubing extends AbstractApplication {
}
}, UnsignedBytes.lexicographicalComparator()).mapPartitions(new FlatMapFunction<Iterator<Tuple2<byte[], byte[]>>, Tuple2<byte[], byte[]>>() {
@Override
- public Iterable<Tuple2<byte[], byte[]>> call(final Iterator<Tuple2<byte[], byte[]>> tuple2Iterator) throws Exception {
+ public Iterator<Tuple2<byte[], byte[]>> call(final Iterator<Tuple2<byte[], byte[]>> tuple2Iterator) throws Exception {
return new Iterable<Tuple2<byte[], byte[]>>() {
final BufferedMeasureCodec codec = new BufferedMeasureCodec(dataTypes);
final Object[] input = new Object[measureSize];
@@ -458,7 +458,7 @@ public class SparkCubing extends AbstractApplication {
}
});
}
- };
+ }.iterator();
}
}, true).mapToPair(new PairFunction<Tuple2<byte[], byte[]>, ImmutableBytesWritable, KeyValue>() {
@Override
@@ -549,7 +549,7 @@ public class SparkCubing extends AbstractApplication {
JavaSparkContext sc = new JavaSparkContext(conf);
HiveContext sqlContext = new HiveContext(sc.sc());
- final DataFrame intermediateTable = sqlContext.sql("select * from " + hiveTable);
+ final Dataset<Row> intermediateTable = sqlContext.sql("select * from " + hiveTable);
final String cubeName = optionsHelper.getOptionValue(OPTION_CUBE_NAME);
final String segmentId = optionsHelper.getOptionValue(OPTION_SEGMENT_ID);
final String confPath = optionsHelper.getOptionValue(OPTION_CONF_PATH);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
index 91aa9f7..587ff78 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubingByLayer.java
@@ -59,7 +59,7 @@ import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.broadcast.Broadcast;
-import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.hive.HiveContext;
import org.apache.spark.storage.StorageLevel;
@@ -73,6 +73,7 @@ import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Iterator;
import java.util.List;
@@ -155,7 +156,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
final KylinConfig envConfig = KylinConfig.getInstanceFromEnv();
HiveContext sqlContext = new HiveContext(sc.sc());
- final DataFrame intermediateTable = sqlContext.table(hiveTable);
+ final Dataset<Row> intermediateTable = sqlContext.table(hiveTable);
final CubeInstance cubeInstance = CubeManager.getInstance(envConfig).getCube(cubeName);
final CubeDesc cubeDesc = cubeInstance.getDescriptor();
@@ -354,7 +355,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
}
@Override
- public Iterable<Tuple2<ByteArray, Object[]>> call(Tuple2<ByteArray, Object[]> tuple2) throws Exception {
+ public Iterator<Tuple2<ByteArray, Object[]>> call(Tuple2<ByteArray, Object[]> tuple2) throws Exception {
if (initialized == false) {
prepare();
initialized = true;
@@ -368,7 +369,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
// if still empty or null
if (myChildren == null || myChildren.size() == 0) {
- return EMTPY_ITERATOR;
+ return EMTPY_ITERATOR.iterator();
}
List<Tuple2<ByteArray, Object[]>> tuples = new ArrayList(myChildren.size());
@@ -382,7 +383,7 @@ public class SparkCubingByLayer extends AbstractApplication implements Serializa
tuples.add(new Tuple2<>(new ByteArray(newKey), tuple2._2()));
}
- return tuples;
+ return tuples.iterator();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkHiveDemo.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkHiveDemo.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkHiveDemo.java
index e1ba470..58d4222 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkHiveDemo.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkHiveDemo.java
@@ -22,7 +22,8 @@ import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
import org.apache.spark.sql.hive.HiveContext;
/**
@@ -45,7 +46,7 @@ public class SparkHiveDemo extends AbstractApplication {
SparkConf conf = new SparkConf().setAppName("Simple Application");
JavaSparkContext sc = new JavaSparkContext(conf);
HiveContext sqlContext = new HiveContext(sc.sc());
- final DataFrame dataFrame = sqlContext.sql("select * from test_kylin_fact");
+ final Dataset<Row> dataFrame = sqlContext.sql("select * from test_kylin_fact");
System.out.println("count * of the table:" + dataFrame.count());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 6a571df..619bf99 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -170,7 +170,7 @@ kylin.engine.spark.rdd-partition-cut-mb=100
### Spark conf overwrite for cube engine
kylin.engine.spark-conf.spark.yarn.submit.file.replication=1
kylin.engine.spark-conf.spark.master=yarn
-kylin.engine.spark-conf.spark.submit.deployMode=cluster
+#kylin.engine.spark-conf.spark.submit.deployMode=cluster
kylin.engine.spark-conf.spark.yarn.executor.memoryOverhead=384
kylin.engine.spark-conf.spark.yarn.driver.memoryOverhead=256
kylin.engine.spark-conf.spark.executor.memory=768M
@@ -184,7 +184,7 @@ kylin.engine.spark-conf.spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/kylin
kylin.engine.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current
kylin.engine.spark-conf.spark.yarn.am.extraJavaOptions=-Dhdp.version=current
kylin.engine.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current
-
+kylin.engine.spark-conf.spark.hadoop.yarn.timeline-service.enabled=false
### QUERY PUSH DOWN ###
#kylin.query.pushdown.runner-class-name=org.apache.kylin.query.adhoc.PushDownRunnerJdbcImpl
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index e82867f..9583b42 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -233,7 +233,7 @@
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
+ <artifactId>kafka_2.11</artifactId>
<scope>provided</scope>
</dependency>
@@ -249,23 +249,45 @@
<scope>test</scope>
</dependency>
+
<!-- Spark dependency -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
+ <version>${spark.version}</version>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
+ <version>${spark.version}</version>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.10</artifactId>
+ <version>${spark.version}</version>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 506d734..732deac 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,7 +61,7 @@
<kafka.version>0.10.1.0</kafka.version>
<!-- Spark versions -->
- <spark.version>1.6.3</spark.version>
+ <spark.version>2.1.1</spark.version>
<kryo.version>4.0.0</kryo.version>
<!-- <reflections.version>0.9.10</reflections.version> -->
@@ -577,19 +577,19 @@
<!-- Spark dependency -->
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.10</artifactId>
+ <artifactId>spark-core_2.11</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
+ <artifactId>spark-sql_2.11</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.10</artifactId>
+ <artifactId>spark-hive_2.11</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
@@ -602,7 +602,7 @@
<!-- Kafka dependency -->
<dependency>
<groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
+ <artifactId>kafka_2.11</artifactId>
<version>${kafka.version}</version>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index f35cb44..fe82e3f 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -256,7 +256,7 @@
<!-- Spark dependency -->
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.10</artifactId>
+ <artifactId>spark-core_2.11</artifactId>
<scope>provided</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfaeb7e7/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index e4ca76f..9ce665e 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -45,7 +45,7 @@
<!-- Provided -->
<dependency>
<groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
+ <artifactId>kafka_2.11</artifactId>
</dependency>
<!-- Env & Test -->
[03/24] kylin git commit: minor, fix NoSuchMethodException in
org.apache.kylin.rest.service.CubeService#getHTableInfo
Posted by li...@apache.org.
minor, fix NoSuchMethodException in org.apache.kylin.rest.service.CubeService#getHTableInfo
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e58951f3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e58951f3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e58951f3
Branch: refs/heads/2.1.x
Commit: e58951f326adcb946c628b6f2bffd6000f18a280
Parents: 21d25b9
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jul 5 13:14:14 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Jul 5 13:23:28 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/service/CubeService.java | 14 ++++++++------
.../org/apache/kylin/rest/service/HBaseInfoUtil.java | 5 ++---
2 files changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e58951f3/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index aa42cb0..4820ccd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -81,7 +81,8 @@ public class CubeService extends BasicService {
private static final Logger logger = LoggerFactory.getLogger(CubeService.class);
- public static final char[] VALID_CUBENAME = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_".toCharArray();
+ public static final char[] VALID_CUBENAME = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_"
+ .toCharArray();
private WeakHashMap<String, HBaseResponse> htableInfoCache = new WeakHashMap<>();
@@ -101,7 +102,8 @@ public class CubeService extends BasicService {
private AclUtil aclUtil;
@PostFilter(Constant.ACCESS_POST_FILTER_READ)
- public List<CubeInstance> listAllCubes(final String cubeName, final String projectName, final String modelName, boolean exactMatch) {
+ public List<CubeInstance> listAllCubes(final String cubeName, final String projectName, final String modelName,
+ boolean exactMatch) {
List<CubeInstance> cubeInstances = null;
ProjectInstance project = (null != projectName) ? getProjectManager().getProject(projectName) : null;
@@ -128,8 +130,8 @@ public class CubeService extends BasicService {
List<CubeInstance> filterCubes = new ArrayList<CubeInstance>();
for (CubeInstance cubeInstance : filterModelCubes) {
boolean isCubeMatch = (null == cubeName)
- || (!exactMatch && cubeInstance.getName().toLowerCase().contains(cubeName.toLowerCase())) ||
- (exactMatch && cubeInstance.getName().toLowerCase().equals(cubeName.toLowerCase()));
+ || (!exactMatch && cubeInstance.getName().toLowerCase().contains(cubeName.toLowerCase()))
+ || (exactMatch && cubeInstance.getName().toLowerCase().equals(cubeName.toLowerCase()));
if (isCubeMatch) {
filterCubes.add(cubeInstance);
@@ -428,8 +430,8 @@ public class CubeService extends BasicService {
try {
// use reflection to isolate NoClassDef errors when HBase is not available
hr = (HBaseResponse) Class.forName("org.apache.kylin.rest.service.HBaseInfoUtil")//
- .getMethod("getHBaseInfo", new Class[] { String.class, String.class })//
- .invoke(null, new Object[] { tableName, this.getConfig().getStorageUrl() });
+ .getMethod("getHBaseInfo", new Class[] { String.class, KylinConfig.class })//
+ .invoke(null, tableName, this.getConfig());
} catch (Throwable e) {
throw new IOException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e58951f3/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java b/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
index 3f0b2b5..5dd9a0b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
@@ -27,10 +27,9 @@ import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
-/**
- * Created by xiefan on 17-5-5.
- */
public class HBaseInfoUtil {
+
+ @SuppressWarnings("unused") // used by reflection
public static HBaseResponse getHBaseInfo(String tableName, KylinConfig config) throws IOException {
if (!config.getStorageUrl().getScheme().equals("hbase"))
return null;
[22/24] kylin git commit: minor,
save query result by form submit request
Posted by li...@apache.org.
minor, save query result by form submit request
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8e8d43d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8e8d43d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8e8d43d
Branch: refs/heads/2.1.x
Commit: f8e8d43d843296e2bc5cb08ba335a6529598ba89
Parents: 8009c2f
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Jul 25 14:55:33 2017 +0800
Committer: Roger Shi <ro...@hotmail.com>
Committed: Tue Jul 25 15:30:23 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/controller2/QueryControllerV2.java | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e8d43d/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
index 7a74765..5809d6e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
@@ -20,7 +20,9 @@ package org.apache.kylin.rest.controller2;
import java.io.IOException;
import java.sql.SQLException;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -45,6 +47,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.http.MediaType;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
@@ -153,14 +156,17 @@ public class QueryControllerV2 extends BasicController {
}
@RequestMapping(value = "/query/format/{format}", method = RequestMethod.POST, produces = {
- "application/vnd.apache.kylin-v2+json" })
+ "application/vnd.apache.kylin-v2+json" }, consumes = MediaType.APPLICATION_FORM_URLENCODED_VALUE)
@ResponseBody
- public void downloadQueryResultV2(@PathVariable String format, @RequestBody SQLRequest sqlRequest,
+ public void downloadQueryResultV2(@PathVariable String format, SQLRequest sqlRequest,
HttpServletResponse response) {
SQLResponse result = queryService.doQueryWithCache(sqlRequest);
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmssSSS");
+ String nowStr = sdf.format(new Date());
response.setContentType("text/" + format + ";charset=utf-8");
- response.setHeader("Content-Disposition", "attachment; filename=\"result." + format + "\"");
+ response.setHeader("Content-Disposition", "attachment; filename=\"" + nowStr + ".result." + format + "\"");
ICsvListWriter csvWriter = null;
try {
[21/24] kylin git commit: KYLIN-2748 fix:register calcite driver when
the first query
Posted by li...@apache.org.
KYLIN-2748 fix:register calcite driver when the first query
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8009c2f3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8009c2f3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8009c2f3
Branch: refs/heads/2.1.x
Commit: 8009c2f3b4c9a0693dd928ab4b371b738f595fba
Parents: 9141bff
Author: yiming.xu <10...@qq.com>
Authored: Tue Jul 25 13:37:38 2017 +0800
Committer: YIMINGXU <10...@qq.com>
Committed: Tue Jul 25 13:56:50 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/query/QueryConnection.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8009c2f3/query/src/main/java/org/apache/kylin/query/QueryConnection.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/QueryConnection.java b/query/src/main/java/org/apache/kylin/query/QueryConnection.java
index bf8d970..f659533 100644
--- a/query/src/main/java/org/apache/kylin/query/QueryConnection.java
+++ b/query/src/main/java/org/apache/kylin/query/QueryConnection.java
@@ -24,6 +24,7 @@ import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;
+import org.apache.calcite.jdbc.Driver;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
@@ -31,8 +32,13 @@ import org.apache.log4j.Logger;
public class QueryConnection {
private static final Logger logger = Logger.getLogger(QueryConnection.class);
+ private static Boolean isRegister = false;
public static Connection getConnection(String project) throws SQLException {
+ if (!isRegister) {
+ DriverManager.registerDriver(new Driver());
+ isRegister = true;
+ }
File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.getNormalizedProjectName(project),
KylinConfig.getInstanceFromEnv());
Properties info = new Properties();
[24/24] kylin git commit: Merge commit
'aee88fe9415bdf3eb2dd4103fee795b03ebac255' into kap-2.4.x
Posted by li...@apache.org.
Merge commit 'aee88fe9415bdf3eb2dd4103fee795b03ebac255' into kap-2.4.x
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2bd2eb7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2bd2eb7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2bd2eb7
Branch: refs/heads/2.1.x
Commit: e2bd2eb75901fb1dc1d079c1ca14b051344db1bc
Parents: d188ee6 aee88fe
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Jul 25 17:41:27 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Jul 25 17:41:27 2017 +0800
----------------------------------------------------------------------
.../job/execution/DefaultChainedExecutable.java | 20 +++----
.../kylin/job/execution/ExecutableManager.java | 12 +++--
.../apache/kylin/metadata/MetadataManager.java | 1 +
pom.xml | 2 +-
.../query/adhoc/PushDownRunnerJdbcImpl.java | 17 +++---
.../rest/service/AclTableMigrationTool.java | 12 +++--
.../apache/kylin/rest/service/JobService.java | 4 ++
.../apache/kylin/rest/service/QueryService.java | 5 +-
.../apache/kylin/rest/util/PushDownUtil.java | 1 -
server/src/main/resources/kylinSecurity.xml | 2 +
.../kylin/rest/service/JobServiceTest.java | 38 ++++++++++++++
.../apache/kylin/source/jdbc/JdbcExplorer.java | 6 ++-
webapp/app/css/AdminLTE.css | 6 ++-
webapp/app/js/controllers/access.js | 1 -
webapp/app/js/controllers/cubeAdvanceSetting.js | 8 ++-
webapp/app/js/controllers/cubeEdit.js | 52 ++++++++++++++++++
webapp/app/js/controllers/cubeMeasures.js | 52 ++++++++++++++----
webapp/app/js/controllers/modelDataModel.js | 3 +-
webapp/app/js/controllers/sourceMeta.js | 2 -
webapp/app/js/model/cubeDescModel.js | 6 ---
webapp/app/js/model/cubeListModel.js | 1 -
webapp/app/js/model/jobListModel.js | 4 --
webapp/app/js/model/streamingListModel.js | 6 ---
webapp/app/less/layout.less | 2 +-
webapp/app/partials/admin/admin.html | 16 +++---
.../cubeDesigner/advanced_settings.html | 55 ++++++++++++++------
webapp/app/partials/cubeDesigner/info.html | 2 +-
webapp/app/partials/cubeDesigner/measures.html | 18 +++----
webapp/app/partials/cubes/cube_json_edit.html | 2 +-
.../app/partials/modelDesigner/data_model.html | 4 +-
.../app/partials/modelDesigner/model_info.html | 2 +-
.../app/partials/projects/project_create.html | 2 +-
.../app/partials/projects/project_detail.html | 2 +-
33 files changed, 248 insertions(+), 118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2bd2eb7/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
----------------------------------------------------------------------
diff --cc core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
index 4f5c02b,cbd49ae..3efa5a8
mode 100644,100755..100755
--- a/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/DefaultChainedExecutable.java
@@@ -43,7 -43,7 +43,7 @@@ public class DefaultChainedExecutable e
sub.initConfig(config);
}
}
--
++
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
List<? extends Executable> executables = getTasks();
@@@ -58,7 -58,7 +58,8 @@@
// the job is paused
break;
} else if (state == ExecutableState.ERROR) {
-- throw new IllegalStateException("invalid subtask state, subtask:" + subTask.getName() + ", state:" + subTask.getStatus());
++ throw new IllegalStateException(
++ "invalid subtask state, subtask:" + subTask.getName() + ", state:" + subTask.getStatus());
}
if (subTask.isRunnable()) {
return subTask.execute(context);
@@@ -97,7 -88,7 +89,7 @@@
@Override
protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) {
ExecutableManager mgr = getManager();
--
++
if (isDiscarded()) {
setEndTime(System.currentTimeMillis());
notifyUserStatusChange(executableContext, ExecutableState.DISCARDED);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2bd2eb7/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2bd2eb7/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2bd2eb7/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2bd2eb7/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
----------------------------------------------------------------------
diff --cc server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
index 0802718,5d7f47a..c21d5c2
--- a/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/PushDownUtil.java
@@@ -49,16 -49,12 +49,15 @@@ public class PushDownUtil
public static boolean doPushDownQuery(String project, String sql, List<List<String>> results,
List<SelectedColumnMeta> columnMetas, SQLException sqlException) throws Exception {
- boolean isExpectedCause = (ExceptionUtils.getRootCause(sqlException).getClass()
- .equals(NoRealizationFoundException.class));
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ if (!kylinConfig.isPushDownEnabled()) {
+ return false;
+ }
- if (isExpectedCause && kylinConfig.isPushDownEnabled()) {
+ Throwable rootCause = ExceptionUtils.getRootCause(sqlException);
+ boolean isExpectedCause = rootCause != null && (rootCause.getClass().equals(NoRealizationFoundException.class));
+ if (isExpectedCause) {
-
logger.info("Query failed to utilize pre-calculation, routing to other engines", sqlException);
IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(kylinConfig.getPushDownRunnerClassName());
IPushDownConverter converter = (IPushDownConverter) ClassUtil
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2bd2eb7/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
----------------------------------------------------------------------
diff --cc server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
index b47f05f,a509f88..3ad0c0e
--- a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
@@@ -19,12 -19,20 +19,22 @@@
package org.apache.kylin.rest.service;
import java.io.IOException;
+import java.sql.SQLException;
+ import java.util.Collections;
+ import java.util.List;
+ import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.job.constant.JobTimeFilterEnum;
+ import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.exception.JobException;
+ import org.apache.kylin.job.execution.AbstractExecutable;
+ import org.apache.kylin.job.execution.ExecutableContext;
+ import org.apache.kylin.job.execution.ExecutableManager;
+ import org.apache.kylin.job.execution.ExecutableState;
+ import org.apache.kylin.job.execution.ExecuteResult;
+ import org.apache.kylin.job.execution.Output;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.QueryConnection;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
[10/24] kylin git commit: minor, fix model desc NPE in diagnosis,
broken cube can be exported
Posted by li...@apache.org.
minor, fix model desc NPE in diagnosis, broken cube can be exported
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/11aa9ff6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/11aa9ff6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/11aa9ff6
Branch: refs/heads/2.1.x
Commit: 11aa9ff690440d1a8de4785d09a75e94a8633976
Parents: 6252ae3
Author: liveandevil <13...@qq.com>
Authored: Wed Jul 19 20:51:54 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Jul 20 21:59:32 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/tool/CubeMetaExtractor.java | 21 +++++++++++++-------
1 file changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/11aa9ff6/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index ce4ebfa..5bfe96b 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -321,13 +321,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
dealWithStreaming(cube);
- for (TableRef table : modelDesc.getAllTables()) {
- String tableName = table.getTableIdentity();
- addRequired(TableDesc.concatResourcePath(tableName));
- addOptional(TableDesc.concatExdResourcePath(tableName));
- }
-
- addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
+ retrieveDataModelDesc(modelDesc);
addRequired(CubeDesc.concatResourcePath(cubeDesc.getName()));
if (includeSegments) {
@@ -389,6 +383,19 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
}
}
+ private void retrieveDataModelDesc(DataModelDesc modelDesc) {
+ if (modelDesc == null) {
+ return;
+ }
+ for (TableRef table : modelDesc.getAllTables()) {
+ String tableName = table.getTableIdentity();
+ addRequired(TableDesc.concatResourcePath(tableName));
+ addOptional(TableDesc.concatExdResourcePath(tableName));
+ }
+
+ addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
+ }
+
private void addRequired(String record) {
logger.info("adding required resource {}", record);
requiredResources.add(record);
[13/24] kylin git commit: minor,
dirty workaround for dealing with calcite polution in spark cubing
Posted by li...@apache.org.
minor,dirty workaround for dealing with calcite polution in spark cubing
minor,dirty workaround for dealing with calcite polution in spark cubing
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/309fac0e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/309fac0e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/309fac0e
Branch: refs/heads/2.1.x
Commit: 309fac0eec56defc497a81e7e3f6941ac9683bb9
Parents: 1b177a5
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Jul 19 21:00:18 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jul 21 16:13:02 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/metadata/model/ComputedColumnDesc.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/309fac0e/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index 4a50ba7..ab7e074 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -17,15 +17,16 @@
*/
package org.apache.kylin.metadata.model;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
+
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
import java.io.Serializable;
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
-public class ComputedColumnDesc implements Serializable{
+public class ComputedColumnDesc implements Serializable {
@JsonProperty
private String tableIdentity;
@JsonProperty
@@ -46,7 +47,8 @@ public class ComputedColumnDesc implements Serializable{
tableIdentity = tableIdentity.toUpperCase();
columnName = columnName.toUpperCase();
- CalciteParser.ensureNoTableNameExists(expression);
+ if ("true".equals(System.getProperty("needCheckCC")))
+ CalciteParser.ensureNoTableNameExists(expression);
}
public String getFullName() {
[08/24] kylin git commit: KYLIN-2671 fix NPE
Posted by li...@apache.org.
KYLIN-2671 fix NPE
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/24fe4212
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/24fe4212
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/24fe4212
Branch: refs/heads/2.1.x
Commit: 24fe4212b44418266a160b5e70adc49c6eeeb974
Parents: 78018d3
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Jul 18 14:26:05 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Jul 18 14:26:05 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/controller/QueryController.java | 3 ++-
.../java/org/apache/kylin/rest/controller2/QueryControllerV2.java | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/24fe4212/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index b44f66f..44b5e72 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -83,7 +83,8 @@ public class QueryController extends BasicController {
@ResponseBody
public SQLResponse prepareQuery(@RequestBody PrepareSqlRequest sqlRequest) {
Map<String, String> newToggles = Maps.newHashMap();
- newToggles.putAll(sqlRequest.getBackdoorToggles());
+ if (sqlRequest.getBackdoorToggles() != null)
+ newToggles.putAll(sqlRequest.getBackdoorToggles());
newToggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
sqlRequest.setBackdoorToggles(newToggles);
http://git-wip-us.apache.org/repos/asf/kylin/blob/24fe4212/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
index 9343d4f..2ef4415 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
@@ -89,7 +89,8 @@ public class QueryControllerV2 extends BasicController {
@ResponseBody
public EnvelopeResponse prepareQueryV2(@RequestBody PrepareSqlRequest sqlRequest) {
Map<String, String> newToggles = Maps.newHashMap();
- newToggles.putAll(sqlRequest.getBackdoorToggles());
+ if (sqlRequest.getBackdoorToggles() != null)
+ newToggles.putAll(sqlRequest.getBackdoorToggles());
newToggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
sqlRequest.setBackdoorToggles(newToggles);
[09/24] kylin git commit: #1355 bugfix fix swallow exception and exit
the method
Posted by li...@apache.org.
#1355 bugfix fix swallow exception and exit the method
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6252ae34
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6252ae34
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6252ae34
Branch: refs/heads/2.1.x
Commit: 6252ae34564232619181b38fd5fedc952da4382f
Parents: 24fe421
Author: Jiatao Tao <ji...@Kyligences-MacBook-Pro.local>
Authored: Tue Jul 18 20:23:33 2017 +0800
Committer: Jiatao Tao <ji...@Kyligences-MacBook-Pro.local>
Committed: Tue Jul 18 20:39:26 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/query/util/ConvertToComputedColumn.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6252ae34/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
index a9ff360..c9feaab 100644
--- a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
+++ b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
@@ -82,7 +82,8 @@ public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
try {
matchedNodes = getMatchedNodes(inputSql, computedColumn.get(ccExp));
} catch (SqlParseException e) {
- logger.error("Convert to computedColumn Fail,parse sql fail ", e.getMessage());
+ logger.error("Convert to computedColumn Fail,parse sql fail ", e);
+ return inputSql;
}
for (SqlNode node : matchedNodes) {
Pair<Integer, Integer> startEndPos = CalciteParser.getReplacePos(node, lines);
[04/24] kylin git commit: minor,
fix bug "is not null" filter not working on derived dim
Posted by li...@apache.org.
minor, fix bug "is not null" filter not working on derived dim
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8307ff4d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8307ff4d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8307ff4d
Branch: refs/heads/2.1.x
Commit: 8307ff4d398584a616b81271438ecdc78d1f2779
Parents: e58951f
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jul 13 14:18:55 2017 +0800
Committer: Roger Shi <ro...@hotmail.com>
Committed: Thu Jul 13 15:40:10 2017 +0800
----------------------------------------------------------------------
.../gtrecord/GTCubeStorageQueryBase.java | 2 +-
.../resources/query/sql_derived/query13.sql | 26 ++++++++++++++++++++
2 files changed, 27 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8307ff4d/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index f89fc47..10f735e 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -305,7 +305,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
- if (compf.getColumn() == null || compf.getValues().isEmpty())
+ if (compf.getColumn() == null)
return compf;
TblColRef derived = compf.getColumn();
http://git-wip-us.apache.org/repos/asf/kylin/blob/8307ff4d/kylin-it/src/test/resources/query/sql_derived/query13.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_derived/query13.sql b/kylin-it/src/test/resources/query/sql_derived/query13.sql
new file mode 100755
index 0000000..754495f
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_derived/query13.sql
@@ -0,0 +1,26 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name, test_category_groupings.meta_categ_name, sum(test_kylin_fact.price) as gmv, count(*) as trans_cnt
+ from test_kylin_fact
+ inner JOIN edw.test_cal_dt as test_cal_dt
+ ON test_kylin_fact.cal_dt = test_cal_dt.cal_dt
+ inner JOIN test_category_groupings
+ ON test_kylin_fact.leaf_categ_id = test_category_groupings.leaf_categ_id AND test_kylin_fact.lstg_site_id = test_category_groupings.site_id
+ where test_cal_dt.week_beg_dt is not null
+ group by test_cal_dt.week_beg_dt, test_kylin_fact.lstg_format_name, test_category_groupings.meta_categ_name
\ No newline at end of file
[19/24] kylin git commit: Do not allow user "ADMIN" be deleted
Posted by li...@apache.org.
Do not allow user "ADMIN" be deleted
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6214ab9f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6214ab9f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6214ab9f
Branch: refs/heads/2.1.x
Commit: 6214ab9ffa228601248c50bd91423a0abc42028f
Parents: 5cb6aba
Author: nichunen <ch...@kyligence.io>
Authored: Mon Jul 24 21:50:13 2017 +0800
Committer: nichunen <ch...@kyligence.io>
Committed: Mon Jul 24 21:50:13 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/service/UserService.java | 7 +++++-
.../kylin/rest/service/UserServiceTest.java | 24 +++++++++++++++-----
2 files changed, 24 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6214ab9f/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 16fafa3..24e2e30 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -51,10 +51,12 @@ public class UserService implements UserDetailsManager {
public static final String DIR_PREFIX = "/user/";
+ public static final String SUPER_ADMIN = "ADMIN";
+
public static final Serializer<ManagedUser> SERIALIZER = new JsonSerializer<>(ManagedUser.class);
protected ResourceStore aclStore;
-
+
private boolean evictCacheFlag = false;
public boolean isEvictCacheFlag() {
@@ -92,6 +94,9 @@ public class UserService implements UserDetailsManager {
@Override
public void deleteUser(String userName) {
+ if (userName.equals(SUPER_ADMIN))
+ throw new InternalErrorException("User " + userName + " is not allowed to be deleted.");
+
try {
String id = getId(userName);
aclStore.deleteResource(id);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6214ab9f/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
index c49b552..736f9a1 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.security.ManagedUser;
import org.junit.Assert;
import org.junit.Test;
@@ -43,19 +44,19 @@ public class UserServiceTest extends ServiceTestBase {
@Test
public void testBasics() throws IOException {
- userService.deleteUser("ADMIN");
+ userService.deleteUser("MODELER");
- Assert.assertTrue(!userService.userExists("ADMIN"));
+ Assert.assertTrue(!userService.userExists("MODELER"));
List<GrantedAuthority> authorities = new ArrayList<GrantedAuthority>();
authorities.add(new SimpleGrantedAuthority(Constant.ROLE_ADMIN));
- ManagedUser user = new ManagedUser("ADMIN", "PWD", false, authorities);
+ ManagedUser user = new ManagedUser("MODELER", "PWD", false, authorities);
userService.createUser(user);
- Assert.assertTrue(userService.userExists("ADMIN"));
+ Assert.assertTrue(userService.userExists("MODELER"));
- UserDetails ud = userService.loadUserByUsername("ADMIN");
- Assert.assertEquals("ADMIN", ud.getUsername());
+ UserDetails ud = userService.loadUserByUsername("MODELER");
+ Assert.assertEquals("MODELER", ud.getUsername());
Assert.assertEquals("PWD", ud.getPassword());
Assert.assertEquals(Constant.ROLE_ADMIN, ud.getAuthorities().iterator().next().getAuthority());
Assert.assertEquals(1, ud.getAuthorities().size());
@@ -64,4 +65,15 @@ public class UserServiceTest extends ServiceTestBase {
Assert.assertTrue(strings.contains(Constant.ROLE_ADMIN));
}
+
+ @Test
+ public void testDeleteAdmin() throws IOException {
+ try {
+ userService.deleteUser("ADMIN");
+ throw new InternalErrorException();
+ } catch (InternalErrorException e) {
+ Assert.assertEquals(e.getMessage(), "User ADMIN is not allowed to be deleted.");
+ }
+
+ }
}
[07/24] kylin git commit: KYLIN-2743 fix bug
Posted by li...@apache.org.
KYLIN-2743 fix bug
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/78018d33
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/78018d33
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/78018d33
Branch: refs/heads/2.1.x
Commit: 78018d33b7feeab4ad258526461a9c4d7f7961ca
Parents: fdf1389
Author: Li Yang <li...@apache.org>
Authored: Tue Jul 18 12:01:17 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Jul 18 12:01:17 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/metadata/model/TableDesc.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/78018d33/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 530732a..d3543dd 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -75,7 +75,6 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
this.columns[i].init(this);
}
- this.project = other.project;
this.database.setName(other.getDatabase());
this.identity = other.identity;
}
[17/24] kylin git commit: KYLIN-2748 Calcite code generation can not
gc cause OOM
Posted by li...@apache.org.
KYLIN-2748 Calcite code generation can not gc cause OOM
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8898213f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8898213f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8898213f
Branch: refs/heads/2.1.x
Commit: 8898213f9d8229382b805574784e566fb841c7eb
Parents: 057bca4
Author: yiming.xu <10...@qq.com>
Authored: Thu Jul 20 18:12:44 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Mon Jul 24 17:56:04 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/query/QueryConnection.java | 42 ++++++++++++++++++++
.../apache/kylin/rest/service/CacheService.java | 1 +
.../apache/kylin/rest/service/QueryService.java | 10 ++---
.../kylin/rest/service/CubeServiceTest.java | 6 ++-
.../kylin/rest/service/JobServiceTest.java | 6 ++-
.../kylin/rest/service/QueryServiceTest.java | 3 +-
6 files changed, 57 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/query/src/main/java/org/apache/kylin/query/QueryConnection.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/QueryConnection.java b/query/src/main/java/org/apache/kylin/query/QueryConnection.java
new file mode 100644
index 0000000..bf8d970
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/QueryConnection.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.query;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.schema.OLAPSchemaFactory;
+import org.apache.log4j.Logger;
+
+public class QueryConnection {
+ private static final Logger logger = Logger.getLogger(QueryConnection.class);
+
+ public static Connection getConnection(String project) throws SQLException {
+ File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.getNormalizedProjectName(project),
+ KylinConfig.getInstanceFromEnv());
+ Properties info = new Properties();
+ info.put("model", olapTmp.getAbsolutePath());
+ return DriverManager.getConnection("jdbc:calcite:", info);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 742efcf..7758987 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -147,6 +147,7 @@ public class CacheService extends BasicService {
queryDataSource.clearCache();
}
+ @Deprecated
public DataSource getOLAPDataSource(String project) {
project = ProjectInstance.getNormalizedProjectName(project);
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 4061c82..75c16bd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -44,7 +44,6 @@ import java.util.Map;
import java.util.Set;
import javax.annotation.PostConstruct;
-import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
import org.apache.calcite.config.CalciteConnectionConfig;
@@ -81,6 +80,7 @@ import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.metadata.querymeta.TableMetaWithType;
import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.query.QueryConnection;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.constant.Constant;
@@ -517,8 +517,7 @@ public class QueryService extends BasicService {
}
ResultSet JDBCTableMeta = null;
try {
- DataSource dataSource = cacheService.getOLAPDataSource(project);
- conn = dataSource.getConnection();
+ conn = QueryConnection.getConnection(project);
DatabaseMetaData metaData = conn.getMetaData();
JDBCTableMeta = metaData.getTables(null, null, null, null);
@@ -593,8 +592,7 @@ public class QueryService extends BasicService {
}
ResultSet JDBCTableMeta = null;
try {
- DataSource dataSource = cacheService.getOLAPDataSource(project);
- conn = dataSource.getConnection();
+ conn = QueryConnection.getConnection(project);
DatabaseMetaData metaData = conn.getMetaData();
JDBCTableMeta = metaData.getTables(null, null, null, null);
@@ -757,7 +755,7 @@ public class QueryService extends BasicService {
List<SelectedColumnMeta> columnMetas = Lists.newArrayList();
try {
- conn = cacheService.getOLAPDataSource(sqlRequest.getProject()).getConnection();
+ conn = QueryConnection.getConnection(sqlRequest.getProject());
// special case for prepare query.
if (BackdoorToggles.getPrepareOnly()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
index 96146ef..769d9d1 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
@@ -19,11 +19,13 @@
package org.apache.kylin.rest.service;
import java.net.UnknownHostException;
+import java.sql.SQLException;
import java.util.List;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.QueryConnection;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
@@ -45,11 +47,11 @@ public class CubeServiceTest extends ServiceTestBase {
private CacheService cacheService;
@Test
- public void testBasics() throws JsonProcessingException, JobException, UnknownHostException {
+ public void testBasics() throws JsonProcessingException, JobException, UnknownHostException, SQLException {
Assert.assertNotNull(cubeService.getConfig());
Assert.assertNotNull(cubeService.getConfig());
Assert.assertNotNull(cubeService.getMetadataManager());
- Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
+ Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null, true);
Assert.assertNotNull(cubes);
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
index 0932748..b47f05f 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
@@ -19,10 +19,12 @@
package org.apache.kylin.rest.service;
import java.io.IOException;
+import java.sql.SQLException;
import org.apache.kylin.job.constant.JobTimeFilterEnum;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.QueryConnection;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
@@ -42,11 +44,11 @@ public class JobServiceTest extends ServiceTestBase {
private CacheService cacheService;
@Test
- public void testBasics() throws JobException, IOException {
+ public void testBasics() throws JobException, IOException, SQLException {
Assert.assertNotNull(jobService.getConfig());
Assert.assertNotNull(jobService.getConfig());
Assert.assertNotNull(jobService.getMetadataManager());
- Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
+ Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
Assert.assertNull(jobService.getJobInstance("job_not_exist"));
Assert.assertNotNull(jobService.searchJobs(null, null, null, 0, 0, JobTimeFilterEnum.ALL));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/8898213f/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 1e5a7a5..c980959 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -23,6 +23,7 @@ import java.sql.SQLException;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.query.QueryConnection;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.junit.Assert;
@@ -48,7 +49,7 @@ public class QueryServiceTest extends ServiceTestBase {
Assert.assertNotNull(queryService.getConfig());
Assert.assertNotNull(queryService.getConfig());
Assert.assertNotNull(queryService.getMetadataManager());
- Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
+ Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
// Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
//
[16/24] kylin git commit: kap: update find spark_dependency
Posted by li...@apache.org.
kap: update find spark_dependency
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/057bca40
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/057bca40
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/057bca40
Branch: refs/heads/2.1.x
Commit: 057bca400de60606cc69e4befdc16e82337c7c5f
Parents: c53ef3f
Author: yiming.xu <10...@qq.com>
Authored: Mon Jul 24 09:36:24 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Mon Jul 24 12:48:56 2017 +0800
----------------------------------------------------------------------
build/bin/find-spark-dependency.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/057bca40/build/bin/find-spark-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-spark-dependency.sh b/build/bin/find-spark-dependency.sh
index 4ea5c3e..3565bfc 100755
--- a/build/bin/find-spark-dependency.sh
+++ b/build/bin/find-spark-dependency.sh
@@ -35,10 +35,10 @@ then
spark_home=$KYLIN_HOME/spark
fi
-spark_dependency=`find -L $spark_home -name 'spark-assembly-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'`
+spark_dependency=`find -L $spark_home/jars -name '*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'`
if [ -z "$spark_dependency" ]
then
- quit "spark assembly lib not found"
+ quit "spark jars not found"
else
verbose "spark dependency: $spark_dependency"
export spark_dependency