You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/06/29 05:48:08 UTC
[01/50] kylin git commit: minor, add prepare toggle
Repository: kylin
Updated Branches:
refs/heads/master 2f084601b -> 69532cd5f
minor, add prepare toggle
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b172e0c1
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b172e0c1
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b172e0c1
Branch: refs/heads/master
Commit: b172e0c15802d2e2e085b74f62011f13f55a225a
Parents: fa6d1b3
Author: Li Yang <li...@apache.org>
Authored: Wed Jun 14 20:44:08 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Wed Jun 14 21:10:24 2017 +0800
----------------------------------------------------------------------
.../kylin/common/debug/BackdoorToggles.java | 14 ++++++++
.../apache/kylin/query/ITKylinQueryTest.java | 15 ++++++++
.../org/apache/kylin/query/KylinTestBase.java | 5 ++-
.../kylin/query/enumerator/OLAPQuery.java | 36 ++++++++++++++++++--
4 files changed, 66 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b172e0c1/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 95d5d62..8cb48b6 100644
--- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -106,6 +106,10 @@ public class BackdoorToggles {
return Integer.valueOf(v);
}
+ public static boolean getPrepareOnly() {
+ return getBoolean(DEBUG_TOGGLE_PREPARE_ONLY);
+ }
+
private static String getString(String key) {
Map<String, String> toggles = _backdoorToggles.get();
if (toggles == null) {
@@ -230,6 +234,16 @@ public class BackdoorToggles {
*/
public final static String DEBUG_TOGGLE_DUMPED_PARTITION_DIR = "DEBUG_TOGGLE_DUMPED_PARTITION_DIR";
+ /**
+ * set DEBUG_TOGGLE_PREPARE_ONLY="true" to prepare the sql statement and get its result set metadata
+ *
+ example:(put it into request body)
+ "backdoorToggles": {
+ "DEBUG_TOGGLE_PREPARE_ONLY": "true"
+ }
+ */
+ public final static String DEBUG_TOGGLE_PREPARE_ONLY = "DEBUG_TOGGLE_PREPARE_ONLY";
+
// properties on statement may go with this "channel" too
/**
* set ATTR_STATEMENT_MAX_ROWS="maxRows" to statement's max rows property
http://git-wip-us.apache.org/repos/asf/kylin/blob/b172e0c1/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 55041e3..4ea4497 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -200,6 +200,21 @@ public class ITKylinQueryTest extends KylinTestBase {
}
@Test
+ public void testVerifyCountQueryWithPrepare() throws Exception {
+ try {
+ Map<String, String> toggles = Maps.newHashMap();
+ toggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
+ BackdoorToggles.setToggles(toggles);
+
+ verifyResultRowColCount(getQueryFolderPrefix() + "src/test/resources/query/sql_verifyCount");
+
+ } finally {
+ BackdoorToggles.cleanToggles();
+
+ }
+ }
+
+ @Test
public void testVerifyContentQuery() throws Exception {
verifyResultContent(getQueryFolderPrefix() + "src/test/resources/query/sql_verifyContent");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b172e0c1/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index 0db5388..a05d0c3 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -44,6 +44,7 @@ import java.util.logging.LogManager;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -365,7 +366,9 @@ public class KylinTestBase {
ITable kylinTable = executeQuery(kylinConn, queryName, sql, false);
// compare the result
- if (expectRowCount >= 0)
+ if (BackdoorToggles.getPrepareOnly())
+ Assert.assertEquals(queryName, 0, kylinTable.getRowCount());
+ else if (expectRowCount >= 0)
Assert.assertEquals(queryName, expectRowCount, kylinTable.getRowCount());
if (expectColCount >= 0)
http://git-wip-us.apache.org/repos/asf/kylin/blob/b172e0c1/query/src/main/java/org/apache/kylin/query/enumerator/OLAPQuery.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPQuery.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPQuery.java
index 8318a07..debc125 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPQuery.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPQuery.java
@@ -22,12 +22,17 @@ import org.apache.calcite.DataContext;
import org.apache.calcite.linq4j.AbstractEnumerable;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.query.relnode.OLAPContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*/
public class OLAPQuery extends AbstractEnumerable<Object[]> implements Enumerable<Object[]> {
+ private static final Logger logger = LoggerFactory.getLogger(OLAPQuery.class);
+
public enum EnumeratorTypeEnum {
OLAP, //finish query with Cube or II, or a combination of both
LOOKUP_TABLE, //using a snapshot of lookup table
@@ -52,13 +57,38 @@ public class OLAPQuery extends AbstractEnumerable<Object[]> implements Enumerabl
OLAPContext olapContext = OLAPContext.getThreadLocalContextById(contextId);
switch (type) {
case OLAP:
- return new OLAPEnumerator(olapContext, optiqContext);
+ return BackdoorToggles.getPrepareOnly() ? new EmptyEnumerator() : new OLAPEnumerator(olapContext, optiqContext);
case LOOKUP_TABLE:
- return new LookupTableEnumerator(olapContext);
+ return BackdoorToggles.getPrepareOnly() ? new EmptyEnumerator() : new LookupTableEnumerator(olapContext);
case HIVE:
- return new HiveEnumerator(olapContext);
+ return BackdoorToggles.getPrepareOnly() ? new EmptyEnumerator() : new HiveEnumerator(olapContext);
default:
throw new IllegalArgumentException("Wrong type " + type + "!");
}
}
+
+ private static class EmptyEnumerator implements Enumerator<Object[]> {
+
+ EmptyEnumerator() {
+ logger.debug("Using empty enumerator");
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public Object[] current() {
+ return null;
+ }
+
+ @Override
+ public boolean moveNext() {
+ return false;
+ }
+
+ @Override
+ public void reset() {
+ }
+ }
}
[04/50] kylin git commit: minor, odbc only cache success request
Posted by li...@apache.org.
minor, odbc only cache success request
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1b583730
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1b583730
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1b583730
Branch: refs/heads/master
Commit: 1b5837305e9343a3624b50ec460cb88f883236ce
Parents: 27dfdbc
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 15 17:37:38 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Thu Jun 15 17:43:44 2017 +0800
----------------------------------------------------------------------
odbc/Common/QueryCache.cpp | 9 ++++++---
odbc/Common/REST.cpp | 5 ++++-
2 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b583730/odbc/Common/QueryCache.cpp
----------------------------------------------------------------------
diff --git a/odbc/Common/QueryCache.cpp b/odbc/Common/QueryCache.cpp
index 6b49273..5c08d7e 100644
--- a/odbc/Common/QueryCache.cpp
+++ b/odbc/Common/QueryCache.cpp
@@ -42,7 +42,8 @@ const wchar_t* loadCache ( const wchar_t* query )
wstring queryString = wstring(query);
queryString.erase(remove_if(queryString.begin(), queryString.end(), ::isspace), queryString.end());
map<wstring, wstring>::iterator it = queryMap.find(queryString);
- if (it != queryMap.end()) {
+ if (it != queryMap.end())
+ {
return it->second.c_str();
}
return NULL;
@@ -55,11 +56,13 @@ void storeCache (const wchar_t* query, const wchar_t* result)
queryString.erase(remove_if(queryString.begin(), queryString.end(), ::isspace), queryString.end());
map<wstring, wstring>::iterator it = queryMap.find(queryString);
- if (it != queryMap.end()) {
+ if (it != queryMap.end())
+ {
return;
}
- if (queryQueue.size() >= cacheSize) {
+ if (queryQueue.size() >= cacheSize)
+ {
wstring head = queryQueue.front();
queryQueue.pop();
queryMap.erase(head);
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b583730/odbc/Common/REST.cpp
----------------------------------------------------------------------
diff --git a/odbc/Common/REST.cpp b/odbc/Common/REST.cpp
index 2111aca..858ecd4 100644
--- a/odbc/Common/REST.cpp
+++ b/odbc/Common/REST.cpp
@@ -444,7 +444,10 @@ wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* usern
wstring ret = getBodyString ( response );
- storeCache(rawSql, ret.c_str());
+ if (*statusFlag == 1)
+ {
+ storeCache(rawSql, ret.c_str());
+ }
return ret;
}
[48/50] kylin git commit: minor,
add parserTimeStampField to KafkaConfig (#1405)
Posted by li...@apache.org.
minor, add parserTimeStampField to KafkaConfig (#1405)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b2fc2c22
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b2fc2c22
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b2fc2c22
Branch: refs/heads/master
Commit: b2fc2c220ba66482447a7631f90363382bcaa422
Parents: 1acd066
Author: 成 <ch...@kyligence.io>
Authored: Wed Jun 28 11:23:51 2017 +0800
Committer: Billy(Yiming) Liu <li...@gmail.com>
Committed: Wed Jun 28 11:23:51 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/source/kafka/KafkaMRInput.java | 2 +-
.../kylin/source/kafka/config/KafkaConfig.java | 22 ++++++++++++++++++++
2 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2fc2c22/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index 3323afb..5bce4e7 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -103,7 +103,7 @@ public class KafkaMRInput implements IMRInput {
this.cubeSegment = cubeSegment;
this.conf = conf;
try {
- streamingParser = StreamingParser.getStreamingParser(kafkaConfig.getParserName(), kafkaConfig.getParserProperties(), columns);
+ streamingParser = StreamingParser.getStreamingParser(kafkaConfig.getParserName(), kafkaConfig.getAllParserProperties(), columns);
} catch (ReflectiveOperationException e) {
throw new IllegalArgumentException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2fc2c22/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
index a096344..547e738 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/config/KafkaConfig.java
@@ -58,6 +58,9 @@ public class KafkaConfig extends RootPersistentEntity {
@JsonProperty("parserName")
private String parserName;
+ @JsonProperty("parserTimeStampField")
+ private String parserTimeStampField;
+
@Deprecated
@JsonProperty("margin")
private long margin;
@@ -120,6 +123,14 @@ public class KafkaConfig extends RootPersistentEntity {
this.margin = margin;
}
+ public void setParserTimeStampField(String parserTimeStampField) {
+ this.parserTimeStampField = parserTimeStampField;
+ }
+
+ public String getParserTimeStampField() {
+ return this.parserTimeStampField;
+ }
+
public String getParserProperties() {
return parserProperties;
}
@@ -128,6 +139,17 @@ public class KafkaConfig extends RootPersistentEntity {
this.parserProperties = parserProperties;
}
+ public String getAllParserProperties() {
+ StringBuilder sb = new StringBuilder();
+ if (parserProperties != null)
+ sb.append(parserProperties);
+ if (parserTimeStampField != null) {
+ sb.append(";");
+ sb.append(parserTimeStampField);
+ }
+ return sb.toString();
+ }
+
@Override
public KafkaConfig clone() {
try {
[25/50] kylin git commit: minor, move csrf conf to right place
Posted by li...@apache.org.
minor, move csrf conf to right place
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1e386948
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1e386948
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1e386948
Branch: refs/heads/master
Commit: 1e386948ad7381c86891233193a95837b3a47c8b
Parents: 7661ad7
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Jun 20 21:10:13 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Tue Jun 20 21:16:21 2017 +0800
----------------------------------------------------------------------
server/src/main/resources/kylinSecurity.xml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1e386948/server/src/main/resources/kylinSecurity.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/kylinSecurity.xml b/server/src/main/resources/kylinSecurity.xml
index 506b2f1..039bded 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -29,7 +29,6 @@
<scr:expression-handler ref="expressionHandler"/>
</scr:global-method-security>
- <scr:csrf disabled="true"/>
<!-- acl config -->
<bean id="aclPermissionFactory" class="org.apache.kylin.rest.security.AclPermissionFactory"/>
@@ -229,6 +228,7 @@
<beans profile="testing,ldap">
<scr:http auto-config="true" use-expressions="true">
+ <scr:csrf disabled="true"/>
<scr:http-basic entry-point-ref="unauthorisedEntryPoint"/>
<scr:intercept-url pattern="/api/user/authentication*/**" access="permitAll"/>
@@ -270,6 +270,7 @@
<!-- Secured Rest API urls with LDAP basic authentication -->
<scr:http pattern="/api/**" use-expressions="true"
authentication-manager-ref="apiAccessAuthenticationManager">
+ <scr:csrf disabled="true"/>
<scr:http-basic entry-point-ref="unauthorisedEntryPoint"/>
<scr:intercept-url pattern="/api/user/authentication*/**" access="permitAll"/>
@@ -294,6 +295,7 @@
<!-- Secured non-api urls with SAML SSO -->
<scr:http auto-config="true" entry-point-ref="samlEntryPoint" use-expressions="false"
authentication-manager-ref="webAccessAuthenticationManager">
+ <scr:csrf disabled="true"/>
<scr:intercept-url pattern="/**" access="IS_AUTHENTICATED_FULLY"/>
<scr:custom-filter before="FIRST" ref="metadataGeneratorFilter"/>
<scr:custom-filter after="BASIC_AUTH_FILTER" ref="samlFilter"/>
[17/50] kylin git commit: fix sample cube model
Posted by li...@apache.org.
fix sample cube model
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d67bf453
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d67bf453
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d67bf453
Branch: refs/heads/master
Commit: d67bf453aa791f90edd89f1f7a40e8ce6a399312
Parents: 3beef26
Author: Li Yang <li...@apache.org>
Authored: Mon Jun 19 14:11:15 2017 +0800
Committer: Luwei-Chen <ch...@apache.org>
Committed: Mon Jun 19 15:38:01 2017 +0800
----------------------------------------------------------------------
examples/sample_cube/template/model_desc/kylin_sales_model.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d67bf453/examples/sample_cube/template/model_desc/kylin_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/template/model_desc/kylin_sales_model.json b/examples/sample_cube/template/model_desc/kylin_sales_model.json
index 50864fc..bcfc1ec 100644
--- a/examples/sample_cube/template/model_desc/kylin_sales_model.json
+++ b/examples/sample_cube/template/model_desc/kylin_sales_model.json
@@ -58,7 +58,7 @@
"columns" : [ "TRANS_ID", "SELLER_ID", "BUYER_ID", "PART_DT", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION" ]
}, {
"table" : "KYLIN_CAL_DT",
- "columns" : ["CAL_DT", "WEEK_BEG_DT"]
+ "columns" : [ "CAL_DT", "WEEK_BEG_DT", "MONTH_BEG_DT", "YEAR_BEG_DT" ]
}, {
"table" : "KYLIN_CATEGORY_GROUPINGS",
"columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "SITE_ID" ]
[36/50] kylin git commit: #1345 Refine MetaStoreService due to change
of Copy
Posted by li...@apache.org.
#1345 Refine MetaStoreService due to change of Copy
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/22d8faea
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/22d8faea
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/22d8faea
Branch: refs/heads/master
Commit: 22d8faeacdd43d65550e23ce9b8d2bbbb0fd6f96
Parents: e6f58d1
Author: auphyroc99 <45...@qq.com>
Authored: Fri Jun 23 13:07:14 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 23 19:29:08 2017 +0800
----------------------------------------------------------------------
tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/22d8faea/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index bc8f34f..ce4ebfa 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -164,6 +164,8 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
executableDao = ExecutableDao.getInstance(kylinConfig);
realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
badQueryHistoryManager = BadQueryHistoryManager.getInstance(kylinConfig);
+
+ addRequired(ResourceStore.METASTORE_UUID_TAG);
if (optionsHelper.hasOption(OPTION_All_PROJECT)) {
for (ProjectInstance projectInstance : projectManager.listAllProjects()) {
[16/50] kylin git commit: #1089 Clone raw table and scheduler with
cube
Posted by li...@apache.org.
#1089 Clone raw table and scheduler with cube
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3beef268
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3beef268
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3beef268
Branch: refs/heads/master
Commit: 3beef268dd06fb3f275b4accc92836b44d99b175
Parents: 66dc541
Author: Roger Shi <ro...@hotmail.com>
Authored: Mon Jun 19 12:02:35 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Mon Jun 19 13:30:35 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/CubeController.java | 4 +-
.../rest/controller2/CubeControllerV2.java | 41 --------------------
.../apache/kylin/rest/service/CubeService.java | 4 +-
3 files changed, 4 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3beef268/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 14c80a0..6916dd8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.controller;
+import static org.apache.kylin.rest.service.CubeService.VALID_CUBENAME;
+
import java.io.IOException;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -83,8 +85,6 @@ import com.google.common.collect.Maps;
public class CubeController extends BasicController {
private static final Logger logger = LoggerFactory.getLogger(CubeController.class);
- private static final char[] VALID_CUBENAME = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_".toCharArray();
-
@Autowired
@Qualifier("cubeMgmtService")
private CubeService cubeService;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3beef268/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
index 720cf76..93c4d4e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
@@ -25,7 +25,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -43,7 +42,6 @@ import org.apache.kylin.rest.controller.BasicController;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
-import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.request.JobBuildRequest;
import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.response.CubeInstanceResponse;
@@ -80,9 +78,6 @@ import com.google.common.collect.Lists;
public class CubeControllerV2 extends BasicController {
private static final Logger logger = LoggerFactory.getLogger(CubeControllerV2.class);
- public static final char[] VALID_CUBENAME = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_"
- .toCharArray();
-
@Autowired
@Qualifier("cubeMgmtService")
private CubeService cubeService;
@@ -437,42 +432,6 @@ public class CubeControllerV2 extends BasicController {
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, cubeService.purgeCube(cube), "");
}
- @RequestMapping(value = "/{cubeName}/clone", method = { RequestMethod.PUT }, produces = {
- "application/vnd.apache.kylin-v2+json" })
- @ResponseBody
- public EnvelopeResponse cloneCubeV2(@PathVariable String cubeName, @RequestBody CubeRequest cubeRequest)
- throws IOException {
- Message msg = MsgPicker.getMsg();
-
- String newCubeName = cubeRequest.getCubeName();
- String project = cubeRequest.getProject();
-
- CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
- if (cube == null) {
- throw new BadRequestException(String.format(msg.getCUBE_NOT_FOUND(), cubeName));
- }
- if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
- throw new BadRequestException(String.format(msg.getCLONE_BROKEN_CUBE(), cubeName));
- }
- if (!StringUtils.containsOnly(newCubeName, VALID_CUBENAME)) {
- logger.info("Invalid Cube name {}, only letters, numbers and underline supported.", newCubeName);
- throw new BadRequestException(String.format(msg.getINVALID_CUBE_NAME(), cubeName));
- }
-
- CubeDesc cubeDesc = cube.getDescriptor();
- CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
-
- newCubeDesc.setName(newCubeName);
-
- CubeInstance newCube;
- newCube = cubeService.createCubeAndDesc(newCubeName, project, newCubeDesc);
-
- //reload to avoid shallow clone
- cubeService.getCubeDescManager().reloadCubeDescLocal(newCubeName);
-
- return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, newCube, "");
- }
-
@RequestMapping(value = "/{cubeName}/enable", method = { RequestMethod.PUT }, produces = {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
http://git-wip-us.apache.org/repos/asf/kylin/blob/3beef268/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 7f09612..f6de877 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -18,8 +18,6 @@
package org.apache.kylin.rest.service;
-import static org.apache.kylin.rest.controller2.CubeControllerV2.VALID_CUBENAME;
-
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -82,6 +80,8 @@ public class CubeService extends BasicService {
private static final Logger logger = LoggerFactory.getLogger(CubeService.class);
+ public static final char[] VALID_CUBENAME = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_".toCharArray();
+
private WeakHashMap<String, HBaseResponse> htableInfoCache = new WeakHashMap<>();
@Autowired
[02/50] kylin git commit: KYLIN-2632 update get saved queries api
(#1172)
Posted by li...@apache.org.
KYLIN-2632 update get saved queries api (#1172)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3dbdbf54
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3dbdbf54
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3dbdbf54
Branch: refs/heads/master
Commit: 3dbdbf544be012b47d4735a3f5aae0254e97b153
Parents: b172e0c
Author: Luwei-Chen <ch...@apache.org>
Authored: Thu Jun 15 12:22:28 2017 +0800
Committer: luguosheng1314 <55...@qq.com>
Committed: Thu Jun 15 12:22:28 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/rest/controller2/QueryControllerV2.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3dbdbf54/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
index ab4741d..a641a53 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
@@ -109,10 +109,10 @@ public class QueryControllerV2 extends BasicController {
queryService.removeQuery(creator, id);
}
- @RequestMapping(value = "/saved_queries/{project}", method = RequestMethod.GET, produces = {
+ @RequestMapping(value = "/saved_queries", method = RequestMethod.GET, produces = {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
- public EnvelopeResponse getQueriesV2(@PathVariable String project,
+ public EnvelopeResponse getQueriesV2(@RequestParam(value = "project", required = false) String project,
@RequestParam(value = "pageOffset", required = false, defaultValue = "0") Integer pageOffset,
@RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize)
throws IOException {
@@ -121,7 +121,7 @@ public class QueryControllerV2 extends BasicController {
String creator = SecurityContextHolder.getContext().getAuthentication().getName();
List<Query> queries = new ArrayList<Query>();
for (Query query : queryService.getQueries(creator)) {
- if (query.getProject().equals(project))
+ if (project == null || query.getProject().equals(project))
queries.add(query);
}
[03/50] kylin git commit: minor, add cache in ODBC
Posted by li...@apache.org.
minor, add cache in ODBC
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/27dfdbc6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/27dfdbc6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/27dfdbc6
Branch: refs/heads/master
Commit: 27dfdbc65fa68609a8c9fd26c75067d1df4b6753
Parents: 3dbdbf5
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 15 15:12:15 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Thu Jun 15 16:49:07 2017 +0800
----------------------------------------------------------------------
odbc/Common/QueryCache.cpp | 84 +++++++++++++++--------------------------
odbc/Common/QueryCache.h | 1 +
odbc/Common/REST.cpp | 14 +------
3 files changed, 32 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/27dfdbc6/odbc/Common/QueryCache.cpp
----------------------------------------------------------------------
diff --git a/odbc/Common/QueryCache.cpp b/odbc/Common/QueryCache.cpp
index 48c187e..6b49273 100644
--- a/odbc/Common/QueryCache.cpp
+++ b/odbc/Common/QueryCache.cpp
@@ -23,72 +23,48 @@
#include <cpprest/json.h>
#include <cpprest/uri.h>
#include <regex>
+#include <map>
+#include <queue>
+#include <mutex>
#include "Dump.h"
#include "JsonConverter.h"
+using namespace std;
-const wchar_t* alwaysFailQueries[7] = {
- L"(\\s)*CREATE(\\s)*LOCAL(\\s)*TEMPORARY(\\s)*TABLE(\\s)*\"XTableau_B_Connect\"(\\s)*\\((\\s)*\"COL\"(\\s)*INTEGER(\\s)*\\)(\\s)*ON(\\s)*COMMIT(\\s)*PRESERVE(\\s)*ROWS(\\s)*",
- L"(\\s)*DROP(\\s)*TABLE(\\s)*\"XTableau_B_Connect\"(\\s)*",
- L"(\\s)*SELECT(\\s)*TOP(\\s)*1(\\s)*\"COL\"(\\s)*FROM(\\s)*\\(SELECT(\\s)*1(\\s)*AS(\\s)*\"COL\"(\\s)*\\)(\\s)*AS(\\s)*\"CHECKTOP\"(\\s)*",
- L"(\\s)*SELECT(\\s)*\"SUBCOL\"(\\s)*AS(\\s)*\"COL\"(\\s)*FROM(\\s)*\\((\\s)*SELECT(\\s)*1(\\s)*AS(\\s)*\"SUBCOL\"(\\s)*\\)(\\s)*\"SUBQUERY\"(\\s)*GROUP(\\s)*BY(\\s)*1(\\s)*",
- L"(\\s)*SELECT(\\s)*\"SUBCOL\"(\\s)*AS(\\s)*\"COL\"(\\s)*FROM(\\s)*\\((\\s)*SELECT(\\s)*1(\\s)*AS(\\s)*\"SUBCOL\"(\\s)*\\)(\\s)*\"SUBQUERY\"(\\s)*GROUP(\\s)*BY(\\s)*\"COL\"(\\s)*",
- L"(\\s)*INSERT(\\s)*INTO(\\s)*\"XTableau_C_Connect\"(\\s)*SELECT(\\s)*\\*(\\s)*FROM(\\s)*\\(SELECT(\\s)*1(\\s)*AS(\\s)*COL(\\s)*\\)(\\s)*AS(\\s)*CHECKTEMP(\\s)*LIMIT(\\s)*1(\\s)*",
- L"(\\s)*DROP(\\s)*TABLE(\\s)*\"XTableau_C_Connect\"(\\s)*"
-};
+map<wstring, wstring> queryMap;
+queue<wstring> queryQueue;
+const int cacheSize = 20;
+mutex cacheMutex;
-const wchar_t* alwaysSuccessQueries[3] = {
- L"(\\s)*SELECT(\\s)*1(\\s)*",
- L"(\\s)*SELECT(\\s)*\"COL\"(\\s)*FROM(\\s)*\\(SELECT(\\s)*1(\\s)*AS(\\s)*\"COL\"\\)(\\s)*AS(\\s)*\"SUBQUERY\"(\\s)*",
- L"(\\s)*SELECT(\\s)*\"COL\"(\\s)*FROM(\\s)*\\(SELECT(\\s)*1(\\s)*AS(\\s)*\"COL\"\\)(\\s)*AS(\\s)*\"CHECKTOP\"(\\s)*LIMIT(\\s)*1(\\s)*"
-};
-
-const wchar_t* alwaysSuccessResults[3] = {
- L"{\"columnMetas\":[{\"isNullable\":2,\"displaySize\":11,\"label\":\"COL\",\"name\":\"COL\",\"schemaName\":\"\",\"catelogName\":\"\",\"tableName\":\"\",\"precision\":10,\"scale\":0,\"columnType\":4,\"columnTypeName\":\"int4\",\"writable\":true,\"caseSensitive\":false,\"autoIncrement\":false,\"searchable\":true,\"currency\":false,\"signed\":true,\"definitelyWritable\":false,\"readOnly\":false}],\"results\":[[\"1\"]],\"isResultsFlatten\":false,\"flattenResult\":null,\"flattenResultOriginalSize\":0,\"cubes\":null,\"affectedRowCount\":0,\"isException\":false,\"exceptionMessage\":null,\"duration\":0.002,\"partial\":false}",
- L"{\"columnMetas\":[{\"isNullable\":2,\"displaySize\":11,\"label\":\"COL\",\"name\":\"COL\",\"schemaName\":\"\",\"catelogName\":\"\",\"tableName\":\"\",\"precision\":10,\"scale\":0,\"columnType\":4,\"columnTypeName\":\"int4\",\"writable\":true,\"caseSensitive\":false,\"autoIncrement\":false,\"searchable\":true,\"currency\":false,\"signed\":true,\"definitelyWritable\":false,\"readOnly\":false}],\"results\":[[\"1\"]],\"isResultsFlatten\":false,\"flattenResult\":null,\"flattenResultOriginalSize\":0,\"cubes\":null,\"affectedRowCount\":0,\"isException\":false,\"exceptionMessage\":null,\"duration\":0.002,\"partial\":false}",
- L"{\"columnMetas\":[{\"isNullable\":2,\"displaySize\":11,\"label\":\"COL\",\"name\":\"COL\",\"schemaName\":\"\",\"catelogName\":\"\",\"tableName\":\"\",\"precision\":10,\"scale\":0,\"columnType\":4,\"columnTypeName\":\"int4\",\"writable\":true,\"caseSensitive\":false,\"autoIncrement\":false,\"searchable\":true,\"currency\":false,\"signed\":true,\"definitelyWritable\":false,\"readOnly\":false}],\"results\":[[\"1\"]],\"isResultsFlatten\":false,\"flattenResult\":null,\"flattenResultOriginalSize\":0,\"cubes\":null,\"affectedRowCount\":0,\"isException\":false,\"exceptionMessage\":null,\"duration\":0.002,\"partial\":false}"
-};
-
-int findQuery ( const wchar_t* sql, const wchar_t** regexs, int size )
+const wchar_t* loadCache ( const wchar_t* query )
{
- for ( int i = 0; i < size; ++i )
- {
- std::tr1::wregex rgx ( regexs[i], regex_constants::icase );
- bool match = std::tr1::regex_search ( sql, rgx );
-
- if ( match )
- {
- return i;
- }
+ lock_guard<mutex> lock(cacheMutex);
+ wstring queryString = wstring(query);
+ queryString.erase(remove_if(queryString.begin(), queryString.end(), ::isspace), queryString.end());
+ map<wstring, wstring>::iterator it = queryMap.find(queryString);
+ if (it != queryMap.end()) {
+ return it->second.c_str();
}
-
- return -1;
-}
-
-int findInAlwaysSuccessQuery ( const wchar_t* sql )
-{
- return findQuery ( sql, alwaysSuccessQueries, sizeof ( alwaysSuccessQueries ) / sizeof ( wchar_t*) );
-}
-
-int findInAlwaysFailQuery ( const wchar_t* sql )
-{
- return findQuery ( sql, alwaysFailQueries, sizeof ( alwaysFailQueries ) / sizeof ( wchar_t*) );
+ return NULL;
}
-const wchar_t* loadCache ( const wchar_t* query )
+void storeCache (const wchar_t* query, const wchar_t* result)
{
- int index = 0;
-
- if ( findInAlwaysFailQuery ( query ) >= 0 )
- {
- throw exception ( "Unsupported SQL" );
+ lock_guard<mutex> lock(cacheMutex);
+ wstring queryString = wstring(query);
+ queryString.erase(remove_if(queryString.begin(), queryString.end(), ::isspace), queryString.end());
+
+ map<wstring, wstring>::iterator it = queryMap.find(queryString);
+ if (it != queryMap.end()) {
+ return;
}
- else if ( ( index = findInAlwaysSuccessQuery ( query ) ) >= 0 )
- {
- return alwaysSuccessResults[index];
+ if (queryQueue.size() >= cacheSize) {
+ wstring head = queryQueue.front();
+ queryQueue.pop();
+ queryMap.erase(head);
}
- return NULL;
+ queryQueue.push(queryString);
+ queryMap[queryString] = result;
}
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/27dfdbc6/odbc/Common/QueryCache.h
----------------------------------------------------------------------
diff --git a/odbc/Common/QueryCache.h b/odbc/Common/QueryCache.h
index 1ce6dbd..f06a7ae 100644
--- a/odbc/Common/QueryCache.h
+++ b/odbc/Common/QueryCache.h
@@ -22,4 +22,5 @@
#include "MsgTypes.h"
const wchar_t* loadCache ( const wchar_t* query );
+void storeCache (const wchar_t* query, const wchar_t* result);
http://git-wip-us.apache.org/repos/asf/kylin/blob/27dfdbc6/odbc/Common/REST.cpp
----------------------------------------------------------------------
diff --git a/odbc/Common/REST.cpp b/odbc/Common/REST.cpp
index 8fe62d5..2111aca 100644
--- a/odbc/Common/REST.cpp
+++ b/odbc/Common/REST.cpp
@@ -45,19 +45,6 @@ using namespace concurrency::streams;
using namespace web;
using namespace web::json;
-void printLog ( const char* msg )
-{
- time_t now = time ( 0 );
- struct tm tstruct;
- char buffer[100];
- tstruct = *localtime ( &now );
- strftime ( buffer, 100, "%Y-%m-%d.%X", &tstruct );
- printf ( buffer );
- printf ( "\n" );
- printf ( msg );
- printf ( "\n" );
-}
-
/// <summary>
/// Find the longest length
/// </summary>
@@ -457,6 +444,7 @@ wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* usern
wstring ret = getBodyString ( response );
+ storeCache(rawSql, ret.c_str());
return ret;
}
[41/50] kylin git commit: minor, fix agg combination calcuation
Posted by li...@apache.org.
minor, fix agg combination calcuation
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ac725cbd
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ac725cbd
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ac725cbd
Branch: refs/heads/master
Commit: ac725cbd90976652970d7ce41029e7b9d8cc1852
Parents: 0f8c21b
Author: Roger Shi <ro...@hotmail.com>
Authored: Mon Jun 26 16:42:13 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Mon Jun 26 17:24:29 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/cube/model/AggregationGroup.java | 3 +++
.../java/org/apache/kylin/cube/AggregationGroupRuleTest.java | 5 +++--
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ac725cbd/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index be719be..064d657 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -320,6 +320,9 @@ public class AggregationGroup implements Serializable {
normalDims.removeAll(jointDims);
combination = combination * (1L << normalDims.size());
+ if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid()) {
+ combination += 1;
+ }
combination -= 1; // not include cuboid 0
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/ac725cbd/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index d81d366..ccc3a98 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -58,6 +58,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase {
continue;
}
CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
+ desc.init(getTestConfig());
ValidateContext vContext = new ValidateContext();
rule.validate(desc, vContext);
vContext.print(System.out);
@@ -101,7 +102,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase {
IValidatorRule<CubeDesc> rule = getAggregationGroupRule();
rule.validate(desc, vContext);
vContext.print(System.out);
- assertEquals(0, vContext.getResults().length);
+ assertEquals(1, vContext.getResults().length);
}
@Test
@@ -130,7 +131,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase {
IValidatorRule<CubeDesc> rule = getAggregationGroupRule();
rule.validate(desc, vContext);
vContext.print(System.out);
- assertEquals(1, vContext.getResults().length);
+ assertEquals(2, vContext.getResults().length);
assertEquals("Aggregation group 0 joint dimensions has overlap with more than 1 dimensions in same hierarchy: [CATEG_LVL2_NAME, META_CATEG_NAME]", (vContext.getResults()[0].getMessage()));
}
[50/50] kylin git commit: Merge commit
'2f084601b938d6051c0fe02a9fc075be549547f0' into mergekylin
Posted by li...@apache.org.
Merge commit '2f084601b938d6051c0fe02a9fc075be549547f0' into mergekylin
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/69532cd5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/69532cd5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/69532cd5
Branch: refs/heads/master
Commit: 69532cd5ff3dc7da9a4670795f1230faa489c389
Parents: a0b7e74 2f08460
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jun 29 13:43:13 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Jun 29 13:43:13 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 118 +++----
.../apache/kylin/common/KylinConfigBase.java | 28 ++
.../java/org/apache/kylin/cube/CubeManager.java | 21 +-
.../kylin/cube/cli/DictionaryGeneratorCLI.java | 10 +-
.../InMemCubeBuilderInputConverter.java | 5 +
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
.../apache/kylin/cube/model/RowKeyColDesc.java | 9 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 52 +++-
.../kylin/metadata/model/ISourceAware.java | 1 +
.../kylin/source/ISampleDataDeployer.java | 47 +++
.../java/org/apache/kylin/source/ISource.java | 6 +
.../source/datagen/ModelDataGenerator.java | 7 +-
.../engine/spark/KylinKryoRegistrator.java | 1 +
.../org/apache/kylin/jdbc/KylinConnection.java | 4 +-
.../java/org/apache/kylin/jdbc/DriverTest.java | 10 +
.../kylin/provision/BuildCubeWithEngine.java | 14 +-
.../kylin/source/hive/BeelineHiveClient.java | 2 +-
.../apache/kylin/source/hive/DBConnConf.java | 86 ++++++
.../kylin/source/hive/HiveClientFactory.java | 1 +
.../apache/kylin/source/hive/HiveMRInput.java | 41 ++-
.../kylin/source/hive/HiveMetadataExplorer.java | 78 ++++-
.../apache/kylin/source/hive/HiveSource.java | 17 ++
.../apache/kylin/source/hive/IHiveClient.java | 7 +-
.../org/apache/kylin/source/jdbc/CmdStep.java | 69 +++++
.../apache/kylin/source/jdbc/HiveCmdStep.java | 77 +++++
.../apache/kylin/source/jdbc/JdbcExplorer.java | 305 +++++++++++++++++++
.../kylin/source/jdbc/JdbcHiveMRInput.java | 93 ++++++
.../apache/kylin/source/jdbc/JdbcSource.java | 66 ++++
.../org/apache/kylin/source/jdbc/JdbcTable.java | 67 ++++
.../kylin/source/jdbc/JdbcTableReader.java | 107 +++++++
.../org/apache/kylin/source/jdbc/SqlUtil.java | 107 +++++++
.../apache/kylin/source/kafka/KafkaSource.java | 8 +-
.../kylin/storage/hbase/HBaseResourceStore.java | 7 +-
.../apache/kylin/tool/KylinConfigCLITest.java | 12 +-
webapp/app/js/controllers/modelEdit.js | 5 +-
webapp/app/js/controllers/modelMeasures.js | 33 +-
webapp/app/js/controllers/page.js | 5 +
webapp/app/js/controllers/projects.js | 38 +--
webapp/app/js/controllers/sourceMeta.js | 9 +
webapp/app/js/filters/filter.js | 17 ++
.../cubeDesigner/advanced_settings.html | 2 +-
webapp/app/partials/cubeDesigner/measures.html | 2 +-
webapp/app/partials/cubes/cubes.html | 10 +-
.../app/partials/modelDesigner/data_model.html | 2 +-
.../partials/modelDesigner/model_measures.html | 2 +-
webapp/app/partials/models/model_json_edit.html | 44 +++
webapp/app/partials/models/model_schema.html | 4 +-
webapp/app/partials/models/models_tree.html | 2 +
.../app/partials/projects/project_detail.html | 29 ++
webapp/app/routes.json | 8 +
50 files changed, 1499 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/69532cd5/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/69532cd5/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/69532cd5/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
[10/50] kylin git commit: minor, enable job page search by job name
Posted by li...@apache.org.
minor, enable job page search by job name
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/61825893
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/61825893
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/61825893
Branch: refs/heads/master
Commit: 6182589362b0675db28fa494a0664efd4942e6c3
Parents: 1c3329a
Author: Roger Shi <ro...@hotmail.com>
Authored: Fri Jun 16 17:28:27 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 16 18:08:41 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/JobController.java | 9 ++-
.../kylin/rest/controller2/JobControllerV2.java | 6 +-
.../apache/kylin/rest/service/CubeService.java | 8 +--
.../apache/kylin/rest/service/JobService.java | 60 +++++++++++++++-----
4 files changed, 58 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/61825893/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
index 7c9c40d..749c872 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -72,7 +72,8 @@ public class JobController extends BasicController {
JobTimeFilterEnum timeFilter = JobTimeFilterEnum.getByCode(jobRequest.getTimeFilter());
try {
- jobInstanceList = jobService.searchJobs(jobRequest.getCubeName(), jobRequest.getProjectName(), statusList, jobRequest.getLimit(), jobRequest.getOffset(), timeFilter);
+ jobInstanceList = jobService.searchJobs(jobRequest.getCubeName(), jobRequest.getProjectName(), statusList,
+ jobRequest.getLimit(), jobRequest.getOffset(), timeFilter);
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e);
@@ -106,7 +107,8 @@ public class JobController extends BasicController {
* @return
* @throws IOException
*/
- @RequestMapping(value = "/{jobId}/steps/{stepId}/output", method = { RequestMethod.GET }, produces = { "application/json" })
+ @RequestMapping(value = "/{jobId}/steps/{stepId}/output", method = { RequestMethod.GET }, produces = {
+ "application/json" })
@ResponseBody
public Map<String, String> getStepOutput(@PathVariable String jobId, @PathVariable String stepId) {
Map<String, String> result = new HashMap<String, String>();
@@ -180,7 +182,8 @@ public class JobController extends BasicController {
* @return
* @throws IOException
*/
- @RequestMapping(value = "/{jobId}/steps/{stepId}/rollback", method = { RequestMethod.PUT }, produces = { "application/json" })
+ @RequestMapping(value = "/{jobId}/steps/{stepId}/rollback", method = { RequestMethod.PUT }, produces = {
+ "application/json" })
@ResponseBody
public JobInstance rollback(@PathVariable String jobId, @PathVariable String stepId) {
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/61825893/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
index 5554852..f9db6d8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/JobControllerV2.java
@@ -107,8 +107,8 @@ public class JobControllerV2 extends BasicController {
@RequestMapping(value = "", method = { RequestMethod.GET }, produces = { "application/vnd.apache.kylin-v2+json" })
@ResponseBody
public EnvelopeResponse listV2(@RequestParam(value = "status", required = false) Integer[] status, //
- @RequestParam(value = "timeFilter", required = true) Integer timeFilter, //
- @RequestParam(value = "cubeName", required = false) String cubeName, //
+ @RequestParam(value = "timeFilter") Integer timeFilter, //
+ @RequestParam(value = "jobName", required = false) String jobName, //
@RequestParam(value = "projectName", required = false) String projectName, //
@RequestParam(value = "pageOffset", required = false, defaultValue = "0") Integer pageOffset, //
@RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize, //
@@ -124,7 +124,7 @@ public class JobControllerV2 extends BasicController {
}
}
- List<JobInstance> jobInstanceList = jobService.searchJobs(cubeName, projectName, statusList,
+ List<JobInstance> jobInstanceList = jobService.searchJobsByJobName(jobName, projectName, statusList,
JobTimeFilterEnum.getByCode(timeFilter));
if (sortby.equals("last_modify")) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/61825893/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 74d8578..7f09612 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -234,7 +234,7 @@ public class CubeService extends BasicService {
throws IOException {
Message msg = MsgPicker.getMsg();
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null,
+ final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null,
EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
throw new BadRequestException(String.format(msg.getDISCARD_JOB_FIRST(), cube.getName()));
@@ -266,7 +266,7 @@ public class CubeService extends BasicService {
public void deleteCube(CubeInstance cube) throws IOException {
Message msg = MsgPicker.getMsg();
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null,
+ final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null,
EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR));
if (!cubingJobs.isEmpty()) {
throw new BadRequestException(String.format(msg.getDISCARD_JOB_FIRST(), cube.getName()));
@@ -361,7 +361,7 @@ public class CubeService extends BasicService {
throw new BadRequestException(String.format(msg.getNO_READY_SEGMENT(), cubeName));
}
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null,
+ final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null,
EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
throw new BadRequestException(msg.getENABLE_WITH_RUNNING_JOB());
@@ -482,7 +482,7 @@ public class CubeService extends BasicService {
}
protected void releaseAllJobs(CubeInstance cube) {
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null);
+ final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cube.getName(), null);
for (CubingJob cubingJob : cubingJobs) {
final ExecutableState status = cubingJob.getStatus();
if (status != ExecutableState.SUCCEED && status != ExecutableState.DISCARDED) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/61825893/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 3db7fb6..7eb1292 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -28,7 +28,10 @@ import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
+import javax.annotation.Nullable;
+
import org.apache.commons.lang3.StringUtils;
+import org.apache.directory.api.util.Strings;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.cube.CubeInstance;
@@ -368,7 +371,7 @@ public class JobService extends BasicService implements InitializingBean {
final JobTimeFilterEnum timeFilter) {
Integer limit = (null == limitValue) ? 30 : limitValue;
Integer offset = (null == offsetValue) ? 0 : offsetValue;
- List<JobInstance> jobs = searchJobs(cubeNameSubstring, projectName, statusList, timeFilter);
+ List<JobInstance> jobs = searchJobsByCubeName(cubeNameSubstring, projectName, statusList, timeFilter);
Collections.sort(jobs);
if (jobs.size() <= offset) {
@@ -382,20 +385,31 @@ public class JobService extends BasicService implements InitializingBean {
return jobs.subList(offset, offset + limit);
}
- public List<JobInstance> searchJobs(final String cubeNameSubstring, final String projectName,
+ public List<JobInstance> searchJobsByCubeName(final String cubeNameSubstring, final String projectName,
+ final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
+ return innerSearchCubingJobs(cubeNameSubstring, null, projectName, statusList, timeFilter);
+ }
+
+ public List<JobInstance> searchJobsByJobName(final String jobName, final String projectName,
final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
+ return innerSearchCubingJobs(null, jobName, projectName, statusList, timeFilter);
+ }
+
+ public List<JobInstance> innerSearchCubingJobs(final String cubeName, final String jobName,
+ final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
+ // prepare time range
Calendar calendar = Calendar.getInstance();
calendar.setTime(new Date());
long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
-
long timeEndInMillis = Long.MAX_VALUE;
Set<ExecutableState> states = convertStatusEnumToStates(statusList);
final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs(timeStartInMillis, timeEndInMillis);
+
return Lists
.newArrayList(
FluentIterable
- .from(searchCubingJobs(cubeNameSubstring, projectName, states, timeStartInMillis,
- timeEndInMillis, allOutputs, false))
+ .from(innerSearchCubingJobs(cubeName, jobName, states, timeStartInMillis,
+ timeEndInMillis, allOutputs, false, projectName))
.transform(new Function<CubingJob, JobInstance>() {
@Override
public JobInstance apply(CubingJob cubingJob) {
@@ -404,9 +418,9 @@ public class JobService extends BasicService implements InitializingBean {
}));
}
- public List<CubingJob> searchCubingJobs(final String cubeName, final String projectName,
+ public List<CubingJob> innerSearchCubingJobs(final String cubeName, final String jobName,
final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis,
- final Map<String, Output> allOutputs, final boolean cubeNameExactMatch) {
+ final Map<String, Output> allOutputs, final boolean nameExactMatch, final String projectName) {
List<CubingJob> results = Lists.newArrayList(FluentIterable.from(
getExecutableManager().getAllAbstractExecutables(timeStartInMillis, timeEndInMillis, CubingJob.class))
.filter(new Predicate<AbstractExecutable>() {
@@ -419,8 +433,8 @@ public class JobService extends BasicService implements InitializingBean {
String executableCubeName = CubingExecutableUtil.getCubeName(executable.getParams());
if (executableCubeName == null)
return true;
- if (cubeNameExactMatch)
- return executableCubeName.equalsIgnoreCase(cubeName);
+ if (nameExactMatch)
+ return executableCubeName.toLowerCase().equals(cubeName);
else
return executableCubeName.toLowerCase().contains(cubeName.toLowerCase());
} else {
@@ -453,19 +467,35 @@ public class JobService extends BasicService implements InitializingBean {
throw e;
}
}
+ }, new Predicate<CubingJob>() {
+ @Override
+ public boolean apply(@Nullable CubingJob cubeJob) {
+ if (cubeJob == null) {
+ return false;
+ }
+
+ if (Strings.isEmpty(jobName)) {
+ return true;
+ }
+
+ if (nameExactMatch) {
+ return cubeJob.getName().toLowerCase().equals(jobName);
+ } else {
+ return cubeJob.getName().toLowerCase().contains(jobName);
+ }
+ }
})));
return results;
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName,
+ public List<CubingJob> listJobsByRealizationName(final String realizationName, final String projectName,
final Set<ExecutableState> statusList) {
- return searchCubingJobs(cubeName, projectName, statusList, 0L, Long.MAX_VALUE,
- getExecutableManager().getAllOutputs(), true);
+ return innerSearchCubingJobs(realizationName, null, statusList, 0L, Long.MAX_VALUE,
+ getExecutableManager().getAllOutputs(), true, projectName);
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName) {
- return searchCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class), 0L, Long.MAX_VALUE,
- getExecutableManager().getAllOutputs(), true);
+ public List<CubingJob> listJobsByRealizationName(final String realizationName, final String projectName) {
+ return listJobsByRealizationName(realizationName, projectName, EnumSet.allOf(ExecutableState.class));
}
}
[42/50] kylin git commit: minor, make MODELER ROLE as a group
Posted by li...@apache.org.
minor, make MODELER ROLE as a group
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f79fa389
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f79fa389
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f79fa389
Branch: refs/heads/master
Commit: f79fa389cc4530950a26a2280811c5929d8956e2
Parents: ac725cb
Author: Roger Shi <ro...@hotmail.com>
Authored: Mon Jun 26 20:07:57 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Mon Jun 26 20:23:10 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/constant/Constant.java | 2 +-
.../src/main/java/org/apache/kylin/rest/service/CubeService.java | 2 +-
.../main/java/org/apache/kylin/rest/service/HybridService.java | 2 +-
.../main/java/org/apache/kylin/rest/service/TableService.java | 4 ++--
4 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f79fa389/server-base/src/main/java/org/apache/kylin/rest/constant/Constant.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/constant/Constant.java b/server-base/src/main/java/org/apache/kylin/rest/constant/Constant.java
index f068e5f..5d326e9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/constant/Constant.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/constant/Constant.java
@@ -35,7 +35,7 @@ public class Constant {
public final static String ROLE_ANALYST = "ROLE_ANALYST";
public final static String ACCESS_HAS_ROLE_ADMIN = "hasRole('ROLE_ADMIN')";
- public final static String ACCESS_HAS_ROLE_MODELER = "hasRole('ROLE_MODELER')";
+// public final static String ACCESS_HAS_ROLE_MODELER = "hasRole('ROLE_MODELER')";
public final static String ACCESS_POST_FILTER_READ = "hasRole('ROLE_ADMIN') or hasPermission(filterObject, 'READ') or hasPermission(filterObject, 'MANAGEMENT') " + "or hasPermission(filterObject, 'OPERATION') or hasPermission(filterObject, 'ADMINISTRATION')";
http://git-wip-us.apache.org/repos/asf/kylin/blob/f79fa389/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index ea5378f..a5fc36a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -153,7 +153,7 @@ public class CubeService extends BasicService {
return getCubeManager().updateCube(cubeBuilder);
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or " + Constant.ACCESS_HAS_ROLE_MODELER)
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public CubeInstance createCubeAndDesc(String cubeName, String projectName, CubeDesc desc) throws IOException {
Message msg = MsgPicker.getMsg();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f79fa389/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
index acb82b5..b718edf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
@@ -41,7 +41,7 @@ public class HybridService extends BasicService {
private static final Logger logger = LoggerFactory.getLogger(HybridService.class);
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or " + Constant.ACCESS_HAS_ROLE_MODELER)
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public HybridInstance createHybridCube(String hybridName, String projectName, String modelName, String[] cubeNames) {
List<String> args = new ArrayList<String>();
args.add("-name");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f79fa389/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index 5cbdb76..915a9be 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -353,7 +353,7 @@ public class TableService extends BasicService {
return descs;
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public void calculateCardinalityIfNotPresent(String[] tables, String submitter) throws Exception {
MetadataManager metaMgr = getMetadataManager();
ExecutableManager exeMgt = ExecutableManager.getInstance(getConfig());
@@ -372,7 +372,7 @@ public class TableService extends BasicService {
*
* @param tableName
*/
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public void calculateCardinality(String tableName, String submitter) throws Exception {
Message msg = MsgPicker.getMsg();
[39/50] kylin git commit: #1358 disable and enable rawtable in cube
controller
Posted by li...@apache.org.
#1358 disable and enable rawtable in cube controller
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81ff9a48
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81ff9a48
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81ff9a48
Branch: refs/heads/master
Commit: 81ff9a488cd94e54d605c74f66a8613010e967fa
Parents: ba7bfd6
Author: Roger Shi <ro...@hotmail.com>
Authored: Fri Jun 23 20:38:55 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 23 21:31:32 2017 +0800
----------------------------------------------------------------------
.../rest/controller2/CubeControllerV2.java | 30 --------------------
1 file changed, 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ff9a48/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
index 63f2d8e..9ffc062 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
@@ -402,22 +402,6 @@ public class CubeControllerV2 extends BasicController {
submitter);
}
- @RequestMapping(value = "/{cubeName}/disable", method = { RequestMethod.PUT }, produces = {
- "application/vnd.apache.kylin-v2+json" })
- @ResponseBody
- public EnvelopeResponse disableCubeV2(@PathVariable String cubeName) throws IOException {
- Message msg = MsgPicker.getMsg();
-
- CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
-
- if (cube == null) {
- throw new BadRequestException(String.format(msg.getCUBE_NOT_FOUND(), cubeName));
- }
-
- return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, cubeService.disableCube(cube), "");
-
- }
-
@RequestMapping(value = "/{cubeName}/purge", method = { RequestMethod.PUT }, produces = {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
@@ -432,20 +416,6 @@ public class CubeControllerV2 extends BasicController {
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, cubeService.purgeCube(cube), "");
}
- @RequestMapping(value = "/{cubeName}/enable", method = { RequestMethod.PUT }, produces = {
- "application/vnd.apache.kylin-v2+json" })
- @ResponseBody
- public EnvelopeResponse enableCubeV2(@PathVariable String cubeName) throws IOException {
- Message msg = MsgPicker.getMsg();
-
- CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
- if (cube == null) {
- throw new BadRequestException(String.format(msg.getCUBE_NOT_FOUND(), cubeName));
- }
-
- return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, cubeService.enableCube(cube), "");
- }
-
/**
* get Hbase Info
*
[34/50] kylin git commit: minor, cube draft provides fuzzy search
Posted by li...@apache.org.
minor, cube draft provides fuzzy search
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c6ade2f0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c6ade2f0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c6ade2f0
Branch: refs/heads/master
Commit: c6ade2f0385f69bd1be90fd57de5e31cffde5f16
Parents: 3f9d158
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 22 13:44:00 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 23 19:14:31 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/controller2/CubeControllerV2.java | 2 +-
.../kylin/rest/controller2/ProjectControllerV2.java | 2 +-
.../java/org/apache/kylin/rest/service/CubeService.java | 10 ++++++----
3 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6ade2f0/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
index 93c4d4e..63f2d8e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
@@ -113,7 +113,7 @@ public class CubeControllerV2 extends BasicController {
}
// draft cubes
- for (Draft d : cubeService.listCubeDrafts(cubeName, modelName, projectName)) {
+ for (Draft d : cubeService.listCubeDrafts(cubeName, modelName, projectName, exactMatch)) {
CubeDesc c = (CubeDesc) d.getEntity();
if (contains(response, c.getName()) == false) {
CubeInstanceResponse r = createCubeInstanceResponseFromDraft(d);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6ade2f0/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
index 5a34807..6dea4e3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
@@ -168,7 +168,7 @@ public class ProjectControllerV2 extends BasicController {
private boolean isProjectEmpty(String projectName) throws IOException {
return cubeService.listAllCubes(projectName).isEmpty()
- && cubeService.listCubeDrafts(null, null, projectName).isEmpty()
+ && cubeService.listCubeDrafts(null, null, projectName, true).isEmpty()
&& modelService.listAllModels(null, projectName, false).isEmpty()
&& modelService.listModelDrafts(null, projectName).isEmpty();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6ade2f0/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index f6de877..ea5378f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -649,21 +649,23 @@ public class CubeService extends BasicService {
}
public Draft getCubeDraft(String cubeName) throws IOException {
- for (Draft d : listCubeDrafts(cubeName, null, null)) {
+ for (Draft d : listCubeDrafts(cubeName, null, null, true)) {
return d;
}
return null;
}
- public List<Draft> listCubeDrafts(String cubeName, String modelName, String project) throws IOException {
+ public List<Draft> listCubeDrafts(String cubeName, String modelName, String project, boolean exactMatch)
+ throws IOException {
List<Draft> result = new ArrayList<>();
for (Draft d : getDraftManager().list(project)) {
RootPersistentEntity e = d.getEntity();
if (e instanceof CubeDesc) {
CubeDesc c = (CubeDesc) e;
- if ((cubeName == null || cubeName.equals(c.getName()))
- && (modelName == null || modelName.equals(c.getModelName()))) {
+ if ((cubeName == null || (exactMatch && cubeName.toLowerCase().equals(c.getName().toLowerCase()))
+ || (!exactMatch && c.getName().toLowerCase().contains(cubeName.toLowerCase())))
+ && (modelName == null || modelName.toLowerCase().equals(c.getModelName().toLowerCase()))) {
result.add(d);
}
}
[14/50] kylin git commit: minor,
fix empty values of SQLPrepare in odbc driver
Posted by li...@apache.org.
minor, fix empty values of SQLPrepare in odbc driver
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ea06950a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ea06950a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ea06950a
Branch: refs/heads/master
Commit: ea06950ae7078ad5c20d334cb14b075868f763b1
Parents: 0ab915d
Author: lidongsjtu <li...@apache.org>
Authored: Sat Jun 17 10:04:57 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Mon Jun 19 13:19:29 2017 +0800
----------------------------------------------------------------------
odbc/Common/REST.cpp | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ea06950a/odbc/Common/REST.cpp
----------------------------------------------------------------------
diff --git a/odbc/Common/REST.cpp b/odbc/Common/REST.cpp
index 8f0c7bb..94e4344 100644
--- a/odbc/Common/REST.cpp
+++ b/odbc/Common/REST.cpp
@@ -133,8 +133,15 @@ void overwrite ( SQLResponse* res )
case ODBCTypes::ODBC_Type_Time :
case ODBCTypes::ODBC_Type_Timestamp :
length = ScanForLength ( res -> results, i );
- meta -> displaySize = length;
- meta -> precision = length;
+ if (length > meta -> displaySize)
+ {
+ meta -> displaySize = length;
+ }
+
+ if (length > meta -> precision)
+ {
+ meta -> precision = length;
+ }
break;
default :
[12/50] kylin git commit: KYLIN-2515 Fix sql convert issue if has the
sql has sub-query
Posted by li...@apache.org.
KYLIN-2515 Fix sql convert issue if has the sql has sub-query
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/70ad90c2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/70ad90c2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/70ad90c2
Branch: refs/heads/master
Commit: 70ad90c2d5ee643b2febde1857a55c150eb3d9cf
Parents: 81d2637
Author: nichunen <ch...@kyligence.io>
Authored: Fri Jun 16 21:05:40 2017 +0800
Committer: 成 <ch...@kyligence.io>
Committed: Fri Jun 16 21:26:56 2017 +0800
----------------------------------------------------------------------
.../source/adhocquery/HiveAdhocConverter.java | 84 +++++++++++++++++---
.../adhocquery/HiveAdhocConverterTest.java | 39 ++++++---
2 files changed, 99 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/70ad90c2/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
index 97d77bf..89b2f7a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/HiveAdhocConverter.java
@@ -26,16 +26,48 @@ import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.ImmutableSet;
+
//TODO: Some workaround ways to make sql readable by hive parser, should replaced it with a more well-designed way
public class HiveAdhocConverter implements IAdHocConverter {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(HiveAdhocConverter.class);
- private static final Pattern EXTRACT_PATTERN = Pattern.compile("\\s+extract\\s*(\\()\\s*(.*?)\\s*from(\\s+)", Pattern.CASE_INSENSITIVE);
- private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\()\\s*select\\s", Pattern.CASE_INSENSITIVE);
- private static final Pattern CAST_PATTERN = Pattern.compile("CAST\\((.*?) (?i)AS\\s*(.*?)\\s*\\)", Pattern.CASE_INSENSITIVE);
- private static final Pattern CONCAT_PATTERN = Pattern.compile("(['_a-z0-9A-Z]+)\\|\\|(['_a-z0-9A-Z]+)", Pattern.CASE_INSENSITIVE);
+ private static final Pattern EXTRACT_PATTERN = Pattern.compile("extract\\s*(\\()\\s*(.*?)\\s*from(\\s+)",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\()\\s*select\\s",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern ALIAS_PATTERN = Pattern.compile("\\s+([`'_a-z0-9A-Z]+)", Pattern.CASE_INSENSITIVE);
+ private static final Pattern CAST_PATTERN = Pattern.compile("CAST\\((.*?) (?i)AS\\s*(.*?)\\s*\\)",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern CONCAT_PATTERN = Pattern.compile("(['_a-z0-9A-Z]+)\\|\\|(['_a-z0-9A-Z]+)",
+ Pattern.CASE_INSENSITIVE);
+ private static final Pattern TIMESTAMPADD_PATTERN = Pattern.compile("timestampadd\\s*\\(\\s*(.*?)\\s*,",
+ Pattern.CASE_INSENSITIVE);
+ private static final ImmutableSet<String> sqlKeyWordsExceptAS = ImmutableSet.of("ABSOLUTE", "ACTION", "ADD", "ALL",
+ "ALLOCATE", "ALTER", "AND", "ANY", "ARE", "ASC", "ASSERTION", "AT", "AUTHORIZATION", "AVG", "BEGIN",
+ "BETWEEN", "BIT", "BIT_LENGTH", "BOTH", "BY", "CASCADE", "CASCADED", "CASE", "CAST", "CATALOG", "CHAR",
+ "CHARACTER", "CHARACTER_LENGTH", "CHAR_LENGTH", "CHECK", "CLOSE", "COALESCE", "COLLATE", "COLLATION",
+ "COLUMN", "COMMIT", "CONNECT", "CONNECTION", "CONSTRAINT", "CONSTRAINTS", "CONTINUE", "CONVERT",
+ "CORRESPONDING", "COUNT", "CREATE", "CROSS", "CURRENT", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP",
+ "CURRENT_USER", "CURSOR", "DATE", "DAY", "DEALLOCATE", "DEC", "DECIMAL", "DECLARE", "DEFAULT", "DEFERRABLE",
+ "DEFERRED", "DELETE", "DESC", "DESCRIBE", "DESCRIPTOR", "DIAGNOSTICS", "DISCONNECT", "DISTINCT", "DOMAIN",
+ "DOUBLE", "DROP", "ELSE", "END", "END-EXEC", "ESCAPE", "EXCEPT", "EXCEPTION", "EXEC", "EXECUTE", "EXISTS",
+ "EXTERNAL", "EXTRACT", "FALSE", "FETCH", "FIRST", "FLOAT", "FOR", "FOREIGN", "FOUND", "FROM", "FULL", "GET",
+ "GLOBAL", "GO", "GOTO", "GRANT", "GROUP", "HAVING", "HOUR", "IDENTITY", "IMMEDIATE", "IN", "INDICATOR",
+ "INITIALLY", "INNER", "INADD", "INSENSITIVE", "INSERT", "INT", "INTEGER", "INTERSECT", "INTERVAL", "INTO",
+ "IS", "ISOLATION", "JOIN", "KEY", "LANGUAGE", "LAST", "LEADING", "LEFT", "LEVEL", "LIKE", "LOCAL", "LOWER",
+ "MATCH", "MAX", "MIN", "MINUTE", "MODULE", "MONTH", "NAMES", "NATIONAL", "NATURAL", "NCHAR", "NEXT", "NO",
+ "NOT", "NULL", "NULLIF", "NUMERIC", "OCTET_LENGTH", "OF", "ON", "ONLY", "OPEN", "OPTION", "OR", "ORDER",
+ "OUTER", "OUTADD", "OVERLAPS", "PAD", "PARTIAL", "POSITION", "PRECISION", "PREPARE", "PRESERVE", "PRIMARY",
+ "PRIOR", "PRIVILEGES", "PROCEDURE", "PUBLIC", "READ", "REAL", "REFERENCES", "RELATIVE", "RESTRICT",
+ "REVOKE", "RIGHT", "ROLLBACK", "ROWS", "SCHEMA", "SCROLL", "SECOND", "SECTION", "SELECT", "SESSION",
+ "SESSION_USER", "SET", "SIZE", "SMALLINT", "SOME", "SPACE", "SQL", "SQLCODE", "SQLERROR", "SQLSTATE",
+ "SUBSTRING", "SUM", "SYSTEM_USER", "TABLE", "TEMPORARY", "THEN", "TIME", "TIMESTAMP", "TIMEZONE_HOUR",
+ "TIMEZONE_MINUTE", "TO", "TRAILING", "TRANSACTION", "TRANSLATE", "TRANSLATION", "TRIM", "TRUE", "UNION",
+ "UNIQUE", "UNKNOWN", "UPDATE", "UPPER", "USAGE", "USER", "USING", "VALUE", "VALUES", "VARCHAR", "VARYING",
+ "VIEW", "WHEN", "WHENEVER", "WHERE", "WITH", "WORK", "WRITE", "YEAR", "ZONE");
public static String replaceString(String originString, String fromString, String toString) {
return originString.replace(fromString, toString);
@@ -55,16 +87,17 @@ public class HiveAdhocConverter implements IAdHocConverter {
int startIdx = extractMatcher.end(3);
int endIdx = parenthesesPairs.get(extractMatcher.start(1));
String extractInner = originString.substring(startIdx, endIdx);
- int originStart = extractMatcher.start(0) + 1;
+ int originStart = extractMatcher.start(0);
int originEnd = endIdx + 1;
- replacedString = replaceString(replacedString, originString.substring(originStart, originEnd), functionStr + "(" + extractInner + ")");
+ replacedString = replaceString(replacedString, originString.substring(originStart, originEnd),
+ functionStr + "(" + extractInner + ")");
}
return replacedString;
}
- public static String castRepalce(String originString) {
+ public static String castReplace(String originString) {
Matcher castMatcher = CAST_PATTERN.matcher(originString);
String replacedString = originString;
@@ -95,7 +128,7 @@ public class HiveAdhocConverter implements IAdHocConverter {
return replacedString;
}
- public static String subqueryRepalce(String originString) {
+ public static String subqueryReplace(String originString) {
Matcher subqueryMatcher = FROM_PATTERN.matcher(originString);
String replacedString = originString;
Map<Integer, Integer> parenthesesPairs = null;
@@ -108,7 +141,30 @@ public class HiveAdhocConverter implements IAdHocConverter {
int startIdx = subqueryMatcher.start(1);
int endIdx = parenthesesPairs.get(startIdx) + 1;
- replacedString = replaceString(replacedString, originString.substring(startIdx, endIdx), originString.substring(startIdx, endIdx) + " as alias");
+ Matcher aliasMatcher = ALIAS_PATTERN.matcher(originString.substring(endIdx));
+ if (aliasMatcher.find()) {
+ String aliasCandidate = aliasMatcher.group(1);
+
+ if (aliasCandidate != null && !sqlKeyWordsExceptAS.contains(aliasCandidate.toUpperCase())) {
+ continue;
+ }
+
+ replacedString = replaceString(replacedString, originString.substring(startIdx, endIdx),
+ originString.substring(startIdx, endIdx) + " as alias");
+ }
+ }
+
+ return replacedString;
+ }
+
+ public static String timestampaddReplace(String originString) {
+ Matcher timestampaddMatcher = TIMESTAMPADD_PATTERN.matcher(originString);
+ String replacedString = originString;
+
+ while (timestampaddMatcher.find()) {
+ String interval = timestampaddMatcher.group(1);
+ String timestampaddStr = replaceString(timestampaddMatcher.group(), interval, "'" + interval + "'");
+ replacedString = replaceString(replacedString, timestampaddMatcher.group(), timestampaddStr);
}
return replacedString;
@@ -121,7 +177,8 @@ public class HiveAdhocConverter implements IAdHocConverter {
while (concatMatcher.find()) {
String leftString = concatMatcher.group(1);
String rightString = concatMatcher.group(2);
- replacedString = replaceString(replacedString, leftString + "||" + rightString, "concat(" + leftString + "," + rightString + ")");
+ replacedString = replaceString(replacedString, leftString + "||" + rightString,
+ "concat(" + leftString + "," + rightString + ")");
}
return replacedString;
@@ -135,10 +192,10 @@ public class HiveAdhocConverter implements IAdHocConverter {
convertedSql = extractReplace(convertedSql);
// Step3.Replace cast type string
- convertedSql = castRepalce(convertedSql);
+ convertedSql = castReplace(convertedSql);
// Step4.Replace sub query
- convertedSql = subqueryRepalce(convertedSql);
+ convertedSql = subqueryReplace(convertedSql);
// Step5.Replace char_length with length
convertedSql = replaceString(convertedSql, "char_length", "length");
@@ -146,6 +203,9 @@ public class HiveAdhocConverter implements IAdHocConverter {
// Step6.Replace "||" with concat
convertedSql = concatReplace(convertedSql);
+ // Step7.Add quote for interval in timestampadd
+ convertedSql = timestampaddReplace(convertedSql);
+
return convertedSql;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/70ad90c2/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java
index 85a6d61..cfb0f32 100644
--- a/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/source/adhocquery/HiveAdhocConverterTest.java
@@ -22,42 +22,57 @@ import org.junit.Test;
import junit.framework.TestCase;
-
public class HiveAdhocConverterTest extends TestCase {
@Test
- public void testSringReplace() {
+ public void testStringReplace() {
String originString = "select count(*) as cnt from test_kylin_fact where char_length(lstg_format_name) < 10";
- String replacedString = HiveAdhocConverter
- .replaceString(originString, "char_length", "length");
- assertEquals(replacedString, "select count(*) as cnt from test_kylin_fact where length(lstg_format_name) < 10");
+ String replacedString = HiveAdhocConverter.replaceString(originString, "char_length", "length");
+ assertEquals("select count(*) as cnt from test_kylin_fact where length(lstg_format_name) < 10", replacedString);
}
@Test
public void testExtractReplace() {
String originString = "ignore EXTRACT(YEAR FROM KYLIN_CAL_DT.CAL_DT) ignore";
String replacedString = HiveAdhocConverter.extractReplace(originString);
- assertEquals(replacedString, "ignore YEAR(KYLIN_CAL_DT.CAL_DT) ignore");
+ assertEquals("ignore YEAR(KYLIN_CAL_DT.CAL_DT) ignore", replacedString);
}
@Test
public void testCastReplace() {
String originString = "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS INTEGER)) ignore";
- String replacedString = HiveAdhocConverter.castRepalce(originString);
- assertEquals(replacedString, "ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS int)) ignore");
+ String replacedString = HiveAdhocConverter.castReplace(originString);
+ assertEquals("ignore EXTRACT(YEAR FROM CAST(KYLIN_CAL_DT.CAL_DT AS int)) ignore", replacedString);
}
@Test
- public void testSubqueryReplace() {
+ public void testSubqueryReplace1() {
String originString = "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) group by seller_id,lstg_format_name";
- String replacedString = HiveAdhocConverter.subqueryRepalce(originString);
- assertEquals(replacedString, "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) as alias group by seller_id,lstg_format_name");
+ String replacedString = HiveAdhocConverter.subqueryReplace(originString);
+ assertEquals(
+ "select seller_id,lstg_format_name,sum(price) from (select * from test_kylin_fact where (lstg_format_name='FP-GTC') limit 20) as alias group by seller_id,lstg_format_name",
+ replacedString);
+ }
+
+ @Test
+ public void testSubqueryReplace2() {
+ String originString = "select count(*) from ( select test_kylin_fact.lstg_format_name from test_kylin_fact where test_kylin_fact.lstg_format_name='FP-GTC' group by test_kylin_fact.lstg_format_name ) t ";
+ String replacedString = HiveAdhocConverter.subqueryReplace(originString);
+ assertEquals(originString, replacedString);
+ }
+
+ @Test
+ public void testSubqueryReplace3() {
+ String originString = "select fact.lstg_format_name from (select * from test_kylin_fact where cal_dt > date'2010-01-01' ) as fact group by fact.lstg_format_name order by CASE WHEN fact.lstg_format_name IS NULL THEN 'sdf' ELSE fact.lstg_format_name END ";
+ String replacedString = HiveAdhocConverter.subqueryReplace(originString);
+ assertEquals(originString, replacedString);
}
@Test
public void testConcatReplace() {
String originString = "select count(*) as cnt from test_kylin_fact where lstg_format_name||'a'='ABINa'";
String replacedString = HiveAdhocConverter.concatReplace(originString);
- assertEquals(replacedString, "select count(*) as cnt from test_kylin_fact where concat(lstg_format_name,'a')='ABINa'");
+ assertEquals("select count(*) as cnt from test_kylin_fact where concat(lstg_format_name,'a')='ABINa'",
+ replacedString);
}
}
[26/50] kylin git commit: Rename stats intermediate table name and
make them droppable (#1267)
Posted by li...@apache.org.
Rename stats intermediate table name and make them droppable (#1267)
* minor, enhance StorageCleanup tool
* #1100, rename stats intermediate table
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bf871691
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bf871691
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bf871691
Branch: refs/heads/master
Commit: bf8716919aaf6bf3880b12fc347aa04d98c6dc83
Parents: 1e38694
Author: 成 <ch...@kyligence.io>
Authored: Wed Jun 21 12:53:52 2017 +0800
Committer: Billy(Yiming) Liu <li...@gmail.com>
Committed: Wed Jun 21 12:53:52 2017 +0800
----------------------------------------------------------------------
.../kylin/engine/mr/common/BatchConstants.java | 3 +-
.../kylin/rest/job/StorageCleanupJob.java | 63 ++++++++++++--------
2 files changed, 40 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf871691/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
index 602b4bb..0cb23ac 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/BatchConstants.java
@@ -45,6 +45,8 @@ public interface BatchConstants {
String CFG_REGION_NUMBER_MAX = "region.number.max";
String CFG_REGION_SPLIT_SIZE = "region.split.size";
String CFG_HFILE_SIZE_GB = "hfile.size.gb";
+ String CFG_STATS_JOB_ID = "stats.job.id";
+ String CFG_STATS_JOB_FREQUENCY = "stats.sample.frequency";
String CFG_KYLIN_LOCAL_TEMP_DIR = "/tmp/kylin/";
String CFG_KYLIN_HDFS_TEMP_DIR = "/tmp/kylin/";
@@ -62,7 +64,6 @@ public interface BatchConstants {
String CFG_OUTPUT_STATISTICS = "statistics";
String CFG_OUTPUT_PARTITION = "partition";
-
/**
* command line ARGuments
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/bf871691/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
index 448e3c6..9b72788 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -60,15 +60,18 @@ import com.google.common.collect.Maps;
public class StorageCleanupJob extends AbstractApplication {
@SuppressWarnings("static-access")
- protected static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete the unused storage").create("delete");
- protected static final Option OPTION_FORCE = OptionBuilder.withArgName("force").hasArg().isRequired(false).withDescription("Warning: will delete all kylin intermediate hive tables").create("force");
+ protected static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false)
+ .withDescription("Delete the unused storage").create("delete");
+ protected static final Option OPTION_FORCE = OptionBuilder.withArgName("force").hasArg().isRequired(false)
+ .withDescription("Warning: will delete all kylin intermediate hive tables").create("force");
protected static final Logger logger = LoggerFactory.getLogger(StorageCleanupJob.class);
public static final int deleteTimeout = 10; // Unit minute
protected boolean delete = false;
protected boolean force = false;
- protected static ExecutableManager executableManager = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
+ protected static ExecutableManager executableManager = ExecutableManager
+ .getInstance(KylinConfig.getInstanceFromEnv());
protected void cleanUnusedHBaseTables() throws IOException {
KylinConfig config = KylinConfig.getInstanceFromEnv();
@@ -76,7 +79,8 @@ public class StorageCleanupJob extends AbstractApplication {
try {
// use reflection to isolate NoClassDef errors when HBase is not available
Class hbaseCleanUpUtil = Class.forName("org.apache.kylin.rest.job.StorageCleanJobHbaseUtil");
- Method cleanUnusedHBaseTables = hbaseCleanUpUtil.getDeclaredMethod("cleanUnusedHBaseTables", boolean.class, int.class);
+ Method cleanUnusedHBaseTables = hbaseCleanUpUtil.getDeclaredMethod("cleanUnusedHBaseTables",
+ boolean.class, int.class);
cleanUnusedHBaseTables.invoke(hbaseCleanUpUtil, delete, deleteTimeout);
} catch (Throwable e) {
throw new IOException(e);
@@ -132,7 +136,8 @@ public class StorageCleanupJob extends AbstractApplication {
if (!state.isFinalState()) {
String path = JobBuilderSupport.getJobWorkingDir(engineConfig.getHdfsWorkingDirectory(), jobId);
allHdfsPathsNeedToBeDeleted.remove(path);
- logger.info("Skip " + path + " from deletion list, as the path belongs to job " + jobId + " with status " + state);
+ logger.info("Skip " + path + " from deletion list, as the path belongs to job " + jobId
+ + " with status " + state);
}
}
@@ -143,7 +148,8 @@ public class StorageCleanupJob extends AbstractApplication {
if (jobUuid != null && jobUuid.equals("") == false) {
String path = JobBuilderSupport.getJobWorkingDir(engineConfig.getHdfsWorkingDirectory(), jobUuid);
allHdfsPathsNeedToBeDeleted.remove(path);
- logger.info("Skip " + path + " from deletion list, as the path belongs to segment " + seg + " of cube " + cube.getName());
+ logger.info("Skip " + path + " from deletion list, as the path belongs to segment " + seg
+ + " of cube " + cube.getName());
}
}
}
@@ -227,22 +233,25 @@ public class StorageCleanupJob extends AbstractApplication {
boolean isNeedDel = true;
- if (line.length() > preFix.length() + uuidLength) {
- String uuid = line.substring(line.length() - uuidLength, line.length());
- uuid = uuid.replace("_", "-");
- final Pattern UUId_PATTERN = Pattern.compile(uuidPattern);
- if (UUId_PATTERN.matcher(uuid).matches()) {
- //Check whether it's a hive table in use
- if (isTableInUse(uuid, workingJobList)) {
- logger.info("Skip deleting because the table is in use");
- isNeedDel = false;
- }
- } else {
- logger.info("Skip deleting because not match pattern");
- isNeedDel = false;
- }
- } else {
- logger.info("Skip deleting because length not qualified");
+ if (line.length() < preFix.length() + uuidLength) {
+ logger.info("Skip deleting because length is not qualified");
+ continue;
+ }
+
+ String uuid = line.substring(line.length() - uuidLength, line.length());
+ uuid = uuid.replace("_", "-");
+ final Pattern UUID_PATTERN = Pattern.compile(uuidPattern);
+
+ if (!UUID_PATTERN.matcher(uuid).matches()) {
+ logger.info("Skip deleting because pattern doesn't match");
+ continue;
+ }
+
+ //Some intermediate table ends with job's uuid
+ if (allJobs.contains(uuid)) {
+ isNeedDel = !workingJobList.contains(uuid);
+ } else if (isTableInUse(uuid, workingJobList)) {
+ logger.info("Skip deleting because the table is in use");
isNeedDel = false;
}
@@ -270,17 +279,21 @@ public class StorageCleanupJob extends AbstractApplication {
String segmentId = uuid.replace("_", "-");
if (segmentId2JobId.containsKey(segmentId)) {
- String path = JobBuilderSupport.getJobWorkingDir(engineConfig.getHdfsWorkingDirectory(), segmentId2JobId.get(segmentId)) + "/" + tableToDelete;
+ String path = JobBuilderSupport.getJobWorkingDir(engineConfig.getHdfsWorkingDirectory(),
+ segmentId2JobId.get(segmentId)) + "/" + tableToDelete;
Path externalDataPath = new Path(path);
FileSystem fs = HadoopUtil.getWorkingFileSystem();
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
logger.info("Hive table {}'s external path {} deleted", tableToDelete, path);
} else {
- logger.info("Hive table {}'s external path {} not exist. It's normal if kylin.source.hive.keep-flat-table set false (By default)", tableToDelete, path);
+ logger.info(
+ "Hive table {}'s external path {} not exist. It's normal if kylin.source.hive.keep-flat-table set false (By default)",
+ tableToDelete, path);
}
} else {
- logger.warn("Hive table {}'s job ID not found, segmentId2JobId: {}", tableToDelete, segmentId2JobId.toString());
+ logger.warn("Hive table {}'s job ID not found, segmentId2JobId: {}", tableToDelete,
+ segmentId2JobId.toString());
}
}
} catch (IOException e) {
[31/50] kylin git commit: #1251 move curator-x-discovery dependency
to ext (out of tomcat classloader)
Posted by li...@apache.org.
#1251 move curator-x-discovery dependency to ext (out of tomcat classloader)
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/401bb0a4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/401bb0a4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/401bb0a4
Branch: refs/heads/master
Commit: 401bb0a466cc3618d6801937d076f05b818d31f4
Parents: cf2f27a
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jun 22 17:08:11 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Thu Jun 22 18:31:21 2017 +0800
----------------------------------------------------------------------
build/script/compress.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/401bb0a4/build/script/compress.sh
----------------------------------------------------------------------
diff --git a/build/script/compress.sh b/build/script/compress.sh
index a72f2f2..cc94f47 100755
--- a/build/script/compress.sh
+++ b/build/script/compress.sh
@@ -34,8 +34,8 @@ package_name=apache-kylin-${version}-bin
cd build/
rm -rf ${package_name}
mkdir ${package_name}
-cp -r lib tool bin conf tomcat spark ../examples/sample_cube commit_SHA1 ${package_name}
-rm -rf lib tomcat spark commit_SHA1
+cp -r ext lib tool bin conf tomcat spark ../examples/sample_cube commit_SHA1 ${package_name}
+rm -rf ext lib tomcat spark commit_SHA1
## comment all default properties, and append them to the user visible kylin.properties
## first 16 lines are license, just skip them
[05/50] kylin git commit: KYLIN-2671 Speed up prepared query execution
Posted by li...@apache.org.
KYLIN-2671 Speed up prepared query execution
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/56e9fb9e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/56e9fb9e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/56e9fb9e
Branch: refs/heads/master
Commit: 56e9fb9e9fa45e9bfb93dd1f4fd0a1027b539c7a
Parents: 1b58373
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jun 15 17:52:22 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Thu Jun 15 18:18:45 2017 +0800
----------------------------------------------------------------------
.../calcite/prepare/CalcitePrepareImpl.java | 1500 ++++++++++++++++++
.../prepare/OnlyPrepareEarlyAbortException.java | 40 +
.../kylin/rest/controller/QueryController.java | 9 +-
.../apache/kylin/rest/service/QueryService.java | 108 +-
4 files changed, 1623 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/56e9fb9e/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java b/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
new file mode 100644
index 0000000..50939b3
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/prepare/CalcitePrepareImpl.java
@@ -0,0 +1,1500 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.calcite.prepare;
+
+import org.apache.calcite.DataContext;
+import org.apache.calcite.adapter.enumerable.EnumerableBindable;
+import org.apache.calcite.adapter.enumerable.EnumerableCalc;
+import org.apache.calcite.adapter.enumerable.EnumerableConvention;
+import org.apache.calcite.adapter.enumerable.EnumerableInterpretable;
+import org.apache.calcite.adapter.enumerable.EnumerableInterpreterRule;
+import org.apache.calcite.adapter.enumerable.EnumerableRel;
+import org.apache.calcite.adapter.enumerable.EnumerableRules;
+import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.avatica.AvaticaParameter;
+import org.apache.calcite.avatica.ColumnMetaData;
+import org.apache.calcite.avatica.Meta;
+import org.apache.calcite.config.CalciteConnectionConfig;
+import org.apache.calcite.interpreter.BindableConvention;
+import org.apache.calcite.interpreter.Bindables;
+import org.apache.calcite.interpreter.Interpreters;
+import org.apache.calcite.jdbc.CalcitePrepare;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.jdbc.CalciteSchema.LatticeEntry;
+import org.apache.calcite.linq4j.Enumerable;
+import org.apache.calcite.linq4j.Linq4j;
+import org.apache.calcite.linq4j.Ord;
+import org.apache.calcite.linq4j.Queryable;
+import org.apache.calcite.linq4j.function.Function1;
+import org.apache.calcite.linq4j.tree.BinaryExpression;
+import org.apache.calcite.linq4j.tree.BlockStatement;
+import org.apache.calcite.linq4j.tree.Blocks;
+import org.apache.calcite.linq4j.tree.ConstantExpression;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.linq4j.tree.MemberExpression;
+import org.apache.calcite.linq4j.tree.MethodCallExpression;
+import org.apache.calcite.linq4j.tree.NewExpression;
+import org.apache.calcite.linq4j.tree.ParameterExpression;
+import org.apache.calcite.materialize.MaterializationService;
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.Convention;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptCostFactory;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.plan.volcano.VolcanoPlanner;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationTraitDef;
+import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.Sort;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule;
+import org.apache.calcite.rel.rules.AggregateReduceFunctionsRule;
+import org.apache.calcite.rel.rules.AggregateStarTableRule;
+import org.apache.calcite.rel.rules.AggregateValuesRule;
+import org.apache.calcite.rel.rules.FilterAggregateTransposeRule;
+import org.apache.calcite.rel.rules.FilterJoinRule;
+import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
+import org.apache.calcite.rel.rules.FilterTableScanRule;
+import org.apache.calcite.rel.rules.JoinAssociateRule;
+import org.apache.calcite.rel.rules.JoinCommuteRule;
+import org.apache.calcite.rel.rules.JoinPushExpressionsRule;
+import org.apache.calcite.rel.rules.JoinPushThroughJoinRule;
+import org.apache.calcite.rel.rules.MaterializedViewFilterScanRule;
+import org.apache.calcite.rel.rules.ProjectFilterTransposeRule;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
+import org.apache.calcite.rel.rules.ProjectTableScanRule;
+import org.apache.calcite.rel.rules.ProjectWindowTransposeRule;
+import org.apache.calcite.rel.rules.ReduceExpressionsRule;
+import org.apache.calcite.rel.rules.SortJoinTransposeRule;
+import org.apache.calcite.rel.rules.SortProjectTransposeRule;
+import org.apache.calcite.rel.rules.SortUnionTransposeRule;
+import org.apache.calcite.rel.rules.TableScanRule;
+import org.apache.calcite.rel.rules.ValuesReduceRule;
+import org.apache.calcite.rel.stream.StreamRules;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexProgram;
+import org.apache.calcite.runtime.Bindable;
+import org.apache.calcite.runtime.Hook;
+import org.apache.calcite.runtime.Typed;
+import org.apache.calcite.schema.Schemas;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.server.CalciteServerStatement;
+import org.apache.calcite.sql.SqlBinaryOperator;
+import org.apache.calcite.sql.SqlExecutableStatement;
+import org.apache.calcite.sql.SqlExplainFormat;
+import org.apache.calcite.sql.SqlExplainLevel;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.SqlUtil;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.parser.SqlParserImplFactory;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.util.ChainedSqlOperatorTable;
+import org.apache.calcite.sql.validate.SqlConformance;
+import org.apache.calcite.sql.validate.SqlValidator;
+import org.apache.calcite.sql2rel.SqlRexConvertletTable;
+import org.apache.calcite.sql2rel.SqlToRelConverter;
+import org.apache.calcite.sql2rel.StandardConvertletTable;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.util.ImmutableIntList;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
+import java.sql.DatabaseMetaData;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.calcite.util.Static.RESOURCE;
+
+/*
+ * OVERRIDE POINT:
+ * - grep KYLIN_ONLY_PREPARE
+ */
+/**
+ * Shit just got real.
+ *
+ * <p>This class is public so that projects that create their own JDBC driver
+ * and server can fine-tune preferences. However, this class and its methods are
+ * subject to change without notice.</p>
+ */
+public class CalcitePrepareImpl implements CalcitePrepare {
+
+ public static final ThreadLocal<Boolean> KYLIN_ONLY_PREPARE = new ThreadLocal<>();
+
+ public static final boolean DEBUG = Util.getBooleanProperty("calcite.debug");
+
+ public static final boolean COMMUTE =
+ Util.getBooleanProperty("calcite.enable.join.commute");
+
+ /** Whether to enable the collation trait. Some extra optimizations are
+ * possible if enabled, but queries should work either way. At some point
+ * this will become a preference, or we will run multiple phases: first
+ * disabled, then enabled. */
+ private static final boolean ENABLE_COLLATION_TRAIT = true;
+
+ /** Whether the bindable convention should be the root convention of any
+ * plan. If not, enumerable convention is the default. */
+ public final boolean enableBindable = Hook.ENABLE_BINDABLE.get(false);
+
+ /** Whether the enumerable convention is enabled. */
+ public static final boolean ENABLE_ENUMERABLE = true;
+
+ /** Whether the streaming is enabled. */
+ public static final boolean ENABLE_STREAM = true;
+
+ private static final Set<String> SIMPLE_SQLS =
+ ImmutableSet.of(
+ "SELECT 1",
+ "select 1",
+ "SELECT 1 FROM DUAL",
+ "select 1 from dual",
+ "values 1",
+ "VALUES 1");
+
+ public static final List<RelOptRule> ENUMERABLE_RULES =
+ ImmutableList.of(
+ EnumerableRules.ENUMERABLE_JOIN_RULE,
+ EnumerableRules.ENUMERABLE_MERGE_JOIN_RULE,
+ EnumerableRules.ENUMERABLE_SEMI_JOIN_RULE,
+ EnumerableRules.ENUMERABLE_CORRELATE_RULE,
+ EnumerableRules.ENUMERABLE_PROJECT_RULE,
+ EnumerableRules.ENUMERABLE_FILTER_RULE,
+ EnumerableRules.ENUMERABLE_AGGREGATE_RULE,
+ EnumerableRules.ENUMERABLE_SORT_RULE,
+ EnumerableRules.ENUMERABLE_LIMIT_RULE,
+ EnumerableRules.ENUMERABLE_COLLECT_RULE,
+ EnumerableRules.ENUMERABLE_UNCOLLECT_RULE,
+ EnumerableRules.ENUMERABLE_UNION_RULE,
+ EnumerableRules.ENUMERABLE_INTERSECT_RULE,
+ EnumerableRules.ENUMERABLE_MINUS_RULE,
+ EnumerableRules.ENUMERABLE_TABLE_MODIFICATION_RULE,
+ EnumerableRules.ENUMERABLE_VALUES_RULE,
+ EnumerableRules.ENUMERABLE_WINDOW_RULE,
+ EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE,
+ EnumerableRules.ENUMERABLE_TABLE_FUNCTION_SCAN_RULE);
+
+ private static final List<RelOptRule> DEFAULT_RULES =
+ ImmutableList.of(
+ AggregateStarTableRule.INSTANCE,
+ AggregateStarTableRule.INSTANCE2,
+ TableScanRule.INSTANCE,
+ COMMUTE
+ ? JoinAssociateRule.INSTANCE
+ : ProjectMergeRule.INSTANCE,
+ FilterTableScanRule.INSTANCE,
+ ProjectFilterTransposeRule.INSTANCE,
+ FilterProjectTransposeRule.INSTANCE,
+ FilterJoinRule.FILTER_ON_JOIN,
+ JoinPushExpressionsRule.INSTANCE,
+ AggregateExpandDistinctAggregatesRule.INSTANCE,
+ AggregateReduceFunctionsRule.INSTANCE,
+ FilterAggregateTransposeRule.INSTANCE,
+ ProjectWindowTransposeRule.INSTANCE,
+ JoinCommuteRule.INSTANCE,
+ JoinPushThroughJoinRule.RIGHT,
+ JoinPushThroughJoinRule.LEFT,
+ SortProjectTransposeRule.INSTANCE,
+ SortJoinTransposeRule.INSTANCE,
+ SortUnionTransposeRule.INSTANCE);
+
+ private static final List<RelOptRule> CONSTANT_REDUCTION_RULES =
+ ImmutableList.of(
+ ReduceExpressionsRule.PROJECT_INSTANCE,
+ ReduceExpressionsRule.FILTER_INSTANCE,
+ ReduceExpressionsRule.CALC_INSTANCE,
+ ReduceExpressionsRule.JOIN_INSTANCE,
+ ValuesReduceRule.FILTER_INSTANCE,
+ ValuesReduceRule.PROJECT_FILTER_INSTANCE,
+ ValuesReduceRule.PROJECT_INSTANCE,
+ AggregateValuesRule.INSTANCE);
+
+ public CalcitePrepareImpl() {
+ }
+
+ public ParseResult parse(
+ Context context, String sql) {
+ return parse_(context, sql, false, false, false);
+ }
+
+ public ConvertResult convert(Context context, String sql) {
+ return (ConvertResult) parse_(context, sql, true, false, false);
+ }
+
+ public AnalyzeViewResult analyzeView(Context context, String sql, boolean fail) {
+ return (AnalyzeViewResult) parse_(context, sql, true, true, fail);
+ }
+
+ /** Shared implementation for {@link #parse}, {@link #convert} and
+ * {@link #analyzeView}. */
+ private ParseResult parse_(Context context, String sql, boolean convert,
+ boolean analyze, boolean fail) {
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ CalciteCatalogReader catalogReader =
+ new CalciteCatalogReader(
+ context.getRootSchema(),
+ context.config().caseSensitive(),
+ context.getDefaultSchemaPath(),
+ typeFactory);
+ SqlParser parser = createParser(sql);
+ SqlNode sqlNode;
+ try {
+ sqlNode = parser.parseStmt();
+ } catch (SqlParseException e) {
+ throw new RuntimeException("parse failed", e);
+ }
+ final SqlValidator validator = createSqlValidator(context, catalogReader);
+ SqlNode sqlNode1 = validator.validate(sqlNode);
+ if (convert) {
+ return convert_(
+ context, sql, analyze, fail, catalogReader, validator, sqlNode1);
+ }
+ return new ParseResult(this, validator, sql, sqlNode1,
+ validator.getValidatedNodeType(sqlNode1));
+ }
+
+ private ParseResult convert_(Context context, String sql, boolean analyze,
+ boolean fail, CalciteCatalogReader catalogReader, SqlValidator validator,
+ SqlNode sqlNode1) {
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ final Convention resultConvention =
+ enableBindable ? BindableConvention.INSTANCE
+ : EnumerableConvention.INSTANCE;
+ final HepPlanner planner = new HepPlanner(new HepProgramBuilder().build());
+ planner.addRelTraitDef(ConventionTraitDef.INSTANCE);
+
+ final SqlToRelConverter.ConfigBuilder configBuilder =
+ SqlToRelConverter.configBuilder().withTrimUnusedFields(true);
+ if (analyze) {
+ configBuilder.withConvertTableAccess(false);
+ }
+
+ final CalcitePreparingStmt preparingStmt =
+ new CalcitePreparingStmt(this, context, catalogReader, typeFactory,
+ context.getRootSchema(), null, planner, resultConvention,
+ createConvertletTable());
+ final SqlToRelConverter converter =
+ preparingStmt.getSqlToRelConverter(validator, catalogReader,
+ configBuilder.build());
+
+ final RelRoot root = converter.convertQuery(sqlNode1, false, true);
+ if (analyze) {
+ return analyze_(validator, sql, sqlNode1, root, fail);
+ }
+ return new ConvertResult(this, validator, sql, sqlNode1,
+ validator.getValidatedNodeType(sqlNode1), root);
+ }
+
+ private AnalyzeViewResult analyze_(SqlValidator validator, String sql,
+ SqlNode sqlNode, RelRoot root, boolean fail) {
+ final RexBuilder rexBuilder = root.rel.getCluster().getRexBuilder();
+ RelNode rel = root.rel;
+ final RelNode viewRel = rel;
+ Project project;
+ if (rel instanceof Project) {
+ project = (Project) rel;
+ rel = project.getInput();
+ } else {
+ project = null;
+ }
+ Filter filter;
+ if (rel instanceof Filter) {
+ filter = (Filter) rel;
+ rel = filter.getInput();
+ } else {
+ filter = null;
+ }
+ TableScan scan;
+ if (rel instanceof TableScan) {
+ scan = (TableScan) rel;
+ } else {
+ scan = null;
+ }
+ if (scan == null) {
+ if (fail) {
+ throw validator.newValidationError(sqlNode,
+ RESOURCE.modifiableViewMustBeBasedOnSingleTable());
+ }
+ return new AnalyzeViewResult(this, validator, sql, sqlNode,
+ validator.getValidatedNodeType(sqlNode), root, null, null, null,
+ null, false);
+ }
+ final RelOptTable targetRelTable = scan.getTable();
+ final RelDataType targetRowType = targetRelTable.getRowType();
+ final Table table = targetRelTable.unwrap(Table.class);
+ final List<String> tablePath = targetRelTable.getQualifiedName();
+ assert table != null;
+ List<Integer> columnMapping;
+ final Map<Integer, RexNode> projectMap = new HashMap<>();
+ if (project == null) {
+ columnMapping = ImmutableIntList.range(0, targetRowType.getFieldCount());
+ } else {
+ columnMapping = new ArrayList<>();
+ for (Ord<RexNode> node : Ord.zip(project.getProjects())) {
+ if (node.e instanceof RexInputRef) {
+ RexInputRef rexInputRef = (RexInputRef) node.e;
+ int index = rexInputRef.getIndex();
+ if (projectMap.get(index) != null) {
+ if (fail) {
+ throw validator.newValidationError(sqlNode,
+ RESOURCE.moreThanOneMappedColumn(
+ targetRowType.getFieldList().get(index).getName(),
+ Util.last(tablePath)));
+ }
+ return new AnalyzeViewResult(this, validator, sql, sqlNode,
+ validator.getValidatedNodeType(sqlNode), root, null, null, null,
+ null, false);
+ }
+ projectMap.put(index, rexBuilder.makeInputRef(viewRel, node.i));
+ columnMapping.add(index);
+ } else {
+ columnMapping.add(-1);
+ }
+ }
+ }
+ final RexNode constraint;
+ if (filter != null) {
+ constraint = filter.getCondition();
+ } else {
+ constraint = rexBuilder.makeLiteral(true);
+ }
+ final List<RexNode> filters = new ArrayList<>();
+ // If we put a constraint in projectMap above, then filters will not be empty despite
+ // being a modifiable view.
+ final List<RexNode> filters2 = new ArrayList<>();
+ boolean retry = false;
+ RelOptUtil.inferViewPredicates(projectMap, filters, constraint);
+ if (fail && !filters.isEmpty()) {
+ final Map<Integer, RexNode> projectMap2 = new HashMap<>();
+ RelOptUtil.inferViewPredicates(projectMap2, filters2, constraint);
+ if (!filters2.isEmpty()) {
+ throw validator.newValidationError(sqlNode,
+ RESOURCE.modifiableViewMustHaveOnlyEqualityPredicates());
+ }
+ retry = true;
+ }
+
+ // Check that all columns that are not projected have a constant value
+ for (RelDataTypeField field : targetRowType.getFieldList()) {
+ final int x = columnMapping.indexOf(field.getIndex());
+ if (x >= 0) {
+ assert Util.skip(columnMapping, x + 1).indexOf(field.getIndex()) < 0
+ : "column projected more than once; should have checked above";
+ continue; // target column is projected
+ }
+ if (projectMap.get(field.getIndex()) != null) {
+ continue; // constant expression
+ }
+ if (field.getType().isNullable()) {
+ continue; // don't need expression for nullable columns; NULL suffices
+ }
+ if (fail) {
+ throw validator.newValidationError(sqlNode,
+ RESOURCE.noValueSuppliedForViewColumn(field.getName(),
+ Util.last(tablePath)));
+ }
+ return new AnalyzeViewResult(this, validator, sql, sqlNode,
+ validator.getValidatedNodeType(sqlNode), root, null, null, null,
+ null, false);
+ }
+
+ final boolean modifiable = filters.isEmpty() || retry && filters2.isEmpty();
+ return new AnalyzeViewResult(this, validator, sql, sqlNode,
+ validator.getValidatedNodeType(sqlNode), root, modifiable ? table : null,
+ ImmutableList.copyOf(tablePath),
+ constraint, ImmutableIntList.copyOf(columnMapping),
+ modifiable);
+ }
+
+ @Override public void executeDdl(Context context, SqlNode node) {
+ if (node instanceof SqlExecutableStatement) {
+ SqlExecutableStatement statement = (SqlExecutableStatement) node;
+ statement.execute(context);
+ return;
+ }
+ throw new UnsupportedOperationException();
+ }
+
+ /** Factory method for default SQL parser. */
+ protected SqlParser createParser(String sql) {
+ return createParser(sql, createParserConfig());
+ }
+
+ /** Factory method for SQL parser with a given configuration. */
+ protected SqlParser createParser(String sql,
+ SqlParser.ConfigBuilder parserConfig) {
+ return SqlParser.create(sql, parserConfig.build());
+ }
+
+ /** Factory method for SQL parser configuration. */
+ protected SqlParser.ConfigBuilder createParserConfig() {
+ return SqlParser.configBuilder();
+ }
+
+ /** Factory method for default convertlet table. */
+ protected SqlRexConvertletTable createConvertletTable() {
+ return StandardConvertletTable.INSTANCE;
+ }
+
+ /** Factory method for cluster. */
+ protected RelOptCluster createCluster(RelOptPlanner planner,
+ RexBuilder rexBuilder) {
+ return RelOptCluster.create(planner, rexBuilder);
+ }
+
+ /** Creates a collection of planner factories.
+ *
+ * <p>The collection must have at least one factory, and each factory must
+ * create a planner. If the collection has more than one planner, Calcite will
+ * try each planner in turn.</p>
+ *
+ * <p>One of the things you can do with this mechanism is to try a simpler,
+ * faster, planner with a smaller rule set first, then fall back to a more
+ * complex planner for complex and costly queries.</p>
+ *
+ * <p>The default implementation returns a factory that calls
+ * {@link #createPlanner(org.apache.calcite.jdbc.CalcitePrepare.Context)}.</p>
+ */
+ protected List<Function1<Context, RelOptPlanner>> createPlannerFactories() {
+ return Collections.<Function1<Context, RelOptPlanner>>singletonList(
+ new Function1<Context, RelOptPlanner>() {
+ public RelOptPlanner apply(Context context) {
+ return createPlanner(context, null, null);
+ }
+ });
+ }
+
+ /** Creates a query planner and initializes it with a default set of
+ * rules. */
+ protected RelOptPlanner createPlanner(CalcitePrepare.Context prepareContext) {
+ return createPlanner(prepareContext, null, null);
+ }
+
+ /** Creates a query planner and initializes it with a default set of
+ * rules. */
+ protected RelOptPlanner createPlanner(
+ final CalcitePrepare.Context prepareContext,
+ org.apache.calcite.plan.Context externalContext,
+ RelOptCostFactory costFactory) {
+ if (externalContext == null) {
+ externalContext = Contexts.of(prepareContext.config());
+ }
+ final VolcanoPlanner planner =
+ new VolcanoPlanner(costFactory, externalContext);
+ planner.addRelTraitDef(ConventionTraitDef.INSTANCE);
+ if (ENABLE_COLLATION_TRAIT) {
+ planner.addRelTraitDef(RelCollationTraitDef.INSTANCE);
+ planner.registerAbstractRelationalRules();
+ }
+ RelOptUtil.registerAbstractRels(planner);
+ for (RelOptRule rule : DEFAULT_RULES) {
+ planner.addRule(rule);
+ }
+ if (prepareContext.config().materializationsEnabled()) {
+ planner.addRule(MaterializedViewFilterScanRule.INSTANCE);
+ }
+ if (enableBindable) {
+ for (RelOptRule rule : Bindables.RULES) {
+ planner.addRule(rule);
+ }
+ }
+ planner.addRule(Bindables.BINDABLE_TABLE_SCAN_RULE);
+ planner.addRule(ProjectTableScanRule.INSTANCE);
+ planner.addRule(ProjectTableScanRule.INTERPRETER);
+
+ if (ENABLE_ENUMERABLE) {
+ for (RelOptRule rule : ENUMERABLE_RULES) {
+ planner.addRule(rule);
+ }
+ planner.addRule(EnumerableInterpreterRule.INSTANCE);
+ }
+
+ if (enableBindable && ENABLE_ENUMERABLE) {
+ planner.addRule(
+ EnumerableBindable.EnumerableToBindableConverterRule.INSTANCE);
+ }
+
+ if (ENABLE_STREAM) {
+ for (RelOptRule rule : StreamRules.RULES) {
+ planner.addRule(rule);
+ }
+ }
+
+ // Change the below to enable constant-reduction.
+ if (false) {
+ for (RelOptRule rule : CONSTANT_REDUCTION_RULES) {
+ planner.addRule(rule);
+ }
+ }
+
+ final SparkHandler spark = prepareContext.spark();
+ if (spark.enabled()) {
+ spark.registerRules(
+ new SparkHandler.RuleSetBuilder() {
+ public void addRule(RelOptRule rule) {
+ // TODO:
+ }
+
+ public void removeRule(RelOptRule rule) {
+ // TODO:
+ }
+ });
+ }
+
+ Hook.PLANNER.run(planner); // allow test to add or remove rules
+
+ return planner;
+ }
+
+ public <T> CalciteSignature<T> prepareQueryable(
+ Context context,
+ Queryable<T> queryable) {
+ return prepare_(context, Query.of(queryable), queryable.getElementType(),
+ -1);
+ }
+
+ public <T> CalciteSignature<T> prepareSql(
+ Context context,
+ Query<T> query,
+ Type elementType,
+ long maxRowCount) {
+ return prepare_(context, query, elementType, maxRowCount);
+ }
+
+ <T> CalciteSignature<T> prepare_(
+ Context context,
+ Query<T> query,
+ Type elementType,
+ long maxRowCount) {
+ if (SIMPLE_SQLS.contains(query.sql)) {
+ return simplePrepare(context, query.sql);
+ }
+
+ if(KYLIN_ONLY_PREPARE.get() != null && KYLIN_ONLY_PREPARE.get()) {
+ ParseResult parseResult = parse(context, query.sql);
+ Class<OnlyPrepareEarlyAbortException> onlyPrepareEarlyAbortExceptionClass =
+ OnlyPrepareEarlyAbortException.class;
+ throw new OnlyPrepareEarlyAbortException(context, parseResult);
+ }
+
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ CalciteCatalogReader catalogReader =
+ new CalciteCatalogReader(
+ context.getRootSchema(),
+ context.config().caseSensitive(),
+ context.getDefaultSchemaPath(),
+ typeFactory);
+ final List<Function1<Context, RelOptPlanner>> plannerFactories =
+ createPlannerFactories();
+ if (plannerFactories.isEmpty()) {
+ throw new AssertionError("no planner factories");
+ }
+ RuntimeException exception = Util.FoundOne.NULL;
+ for (Function1<Context, RelOptPlanner> plannerFactory : plannerFactories) {
+ final RelOptPlanner planner = plannerFactory.apply(context);
+ if (planner == null) {
+ throw new AssertionError("factory returned null planner");
+ }
+ try {
+ return prepare2_(context, query, elementType, maxRowCount,
+ catalogReader, planner);
+ } catch (RelOptPlanner.CannotPlanException e) {
+ exception = e;
+ }
+ }
+ throw exception;
+ }
+
+ /** Quickly prepares a simple SQL statement, circumventing the usual
+ * preparation process. */
+ private <T> CalciteSignature<T> simplePrepare(Context context, String sql) {
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ final RelDataType x =
+ typeFactory.builder()
+ .add(SqlUtil.deriveAliasFromOrdinal(0), SqlTypeName.INTEGER)
+ .build();
+ @SuppressWarnings("unchecked")
+ final List<T> list = (List) ImmutableList.of(1);
+ final List<String> origin = null;
+ final List<List<String>> origins =
+ Collections.nCopies(x.getFieldCount(), origin);
+ final List<ColumnMetaData> columns =
+ getColumnMetaDataList(typeFactory, x, x, origins);
+ final Meta.CursorFactory cursorFactory =
+ Meta.CursorFactory.deduce(columns, null);
+ return new CalciteSignature<>(
+ sql,
+ ImmutableList.<AvaticaParameter>of(),
+ ImmutableMap.<String, Object>of(),
+ x,
+ columns,
+ cursorFactory,
+ ImmutableList.<RelCollation>of(),
+ -1,
+ new Bindable<T>() {
+ public Enumerable<T> bind(DataContext dataContext) {
+ return Linq4j.asEnumerable(list);
+ }
+ },
+ Meta.StatementType.SELECT);
+ }
+
+ /**
+ * Deduces the broad type of statement.
+ * Currently returns SELECT for most statement types, but this may change.
+ *
+ * @param kind Kind of statement
+ */
+ private Meta.StatementType getStatementType(SqlKind kind) {
+ switch (kind) {
+ case INSERT:
+ case DELETE:
+ case UPDATE:
+ return Meta.StatementType.IS_DML;
+ default:
+ return Meta.StatementType.SELECT;
+ }
+ }
+
+ /**
+ * Deduces the broad type of statement for a prepare result.
+ * Currently returns SELECT for most statement types, but this may change.
+ *
+ * @param preparedResult Prepare result
+ */
+ private Meta.StatementType getStatementType(Prepare.PreparedResult preparedResult) {
+ if (preparedResult.isDml()) {
+ return Meta.StatementType.IS_DML;
+ } else {
+ return Meta.StatementType.SELECT;
+ }
+ }
+
+ <T> CalciteSignature<T> prepare2_(
+ Context context,
+ Query<T> query,
+ Type elementType,
+ long maxRowCount,
+ CalciteCatalogReader catalogReader,
+ RelOptPlanner planner) {
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ final EnumerableRel.Prefer prefer;
+ if (elementType == Object[].class) {
+ prefer = EnumerableRel.Prefer.ARRAY;
+ } else {
+ prefer = EnumerableRel.Prefer.CUSTOM;
+ }
+ final Convention resultConvention =
+ enableBindable ? BindableConvention.INSTANCE
+ : EnumerableConvention.INSTANCE;
+ final CalcitePreparingStmt preparingStmt =
+ new CalcitePreparingStmt(this, context, catalogReader, typeFactory,
+ context.getRootSchema(), prefer, planner, resultConvention,
+ createConvertletTable());
+
+ final RelDataType x;
+ final Prepare.PreparedResult preparedResult;
+ final Meta.StatementType statementType;
+ if (query.sql != null) {
+ final CalciteConnectionConfig config = context.config();
+ final SqlParser.ConfigBuilder parserConfig = createParserConfig()
+ .setQuotedCasing(config.quotedCasing())
+ .setUnquotedCasing(config.unquotedCasing())
+ .setQuoting(config.quoting())
+ .setConformance(config.conformance());
+ final SqlParserImplFactory parserFactory =
+ config.parserFactory(SqlParserImplFactory.class, null);
+ if (parserFactory != null) {
+ parserConfig.setParserFactory(parserFactory);
+ }
+ SqlParser parser = createParser(query.sql, parserConfig);
+ SqlNode sqlNode;
+ try {
+ sqlNode = parser.parseStmt();
+ statementType = getStatementType(sqlNode.getKind());
+ } catch (SqlParseException e) {
+ throw new RuntimeException(
+ "parse failed: " + e.getMessage(), e);
+ }
+
+ Hook.PARSE_TREE.run(new Object[] {query.sql, sqlNode});
+
+ if (sqlNode.getKind().belongsTo(SqlKind.DDL)) {
+ executeDdl(context, sqlNode);
+
+ // Return a dummy signature that contains no rows
+ final Bindable<T> bindable = new Bindable<T>() {
+ public Enumerable<T> bind(DataContext dataContext) {
+ return Linq4j.emptyEnumerable();
+ }
+ };
+ return new CalciteSignature<>(query.sql,
+ ImmutableList.<AvaticaParameter>of(),
+ ImmutableMap.<String, Object>of(), null,
+ ImmutableList.<ColumnMetaData>of(), Meta.CursorFactory.OBJECT,
+ ImmutableList.<RelCollation>of(), -1, bindable);
+ }
+
+ final SqlValidator validator =
+ createSqlValidator(context, catalogReader);
+ validator.setIdentifierExpansion(true);
+ validator.setDefaultNullCollation(config.defaultNullCollation());
+
+ preparedResult = preparingStmt.prepareSql(
+ sqlNode, Object.class, validator, true);
+ switch (sqlNode.getKind()) {
+ case INSERT:
+ case DELETE:
+ case UPDATE:
+ case EXPLAIN:
+ // FIXME: getValidatedNodeType is wrong for DML
+ x = RelOptUtil.createDmlRowType(sqlNode.getKind(), typeFactory);
+ break;
+ default:
+ x = validator.getValidatedNodeType(sqlNode);
+ }
+ } else if (query.queryable != null) {
+ x = context.getTypeFactory().createType(elementType);
+ preparedResult =
+ preparingStmt.prepareQueryable(query.queryable, x);
+ statementType = getStatementType(preparedResult);
+ } else {
+ assert query.rel != null;
+ x = query.rel.getRowType();
+ preparedResult = preparingStmt.prepareRel(query.rel);
+ statementType = getStatementType(preparedResult);
+ }
+
+ final List<AvaticaParameter> parameters = new ArrayList<>();
+ final RelDataType parameterRowType = preparedResult.getParameterRowType();
+ for (RelDataTypeField field : parameterRowType.getFieldList()) {
+ RelDataType type = field.getType();
+ parameters.add(
+ new AvaticaParameter(
+ false,
+ getPrecision(type),
+ getScale(type),
+ getTypeOrdinal(type),
+ getTypeName(type),
+ getClassName(type),
+ field.getName()));
+ }
+
+ RelDataType jdbcType = makeStruct(typeFactory, x);
+ final List<List<String>> originList = preparedResult.getFieldOrigins();
+ final List<ColumnMetaData> columns =
+ getColumnMetaDataList(typeFactory, x, jdbcType, originList);
+ Class resultClazz = null;
+ if (preparedResult instanceof Typed) {
+ resultClazz = (Class) ((Typed) preparedResult).getElementType();
+ }
+ final Meta.CursorFactory cursorFactory =
+ preparingStmt.resultConvention == BindableConvention.INSTANCE
+ ? Meta.CursorFactory.ARRAY
+ : Meta.CursorFactory.deduce(columns, resultClazz);
+ //noinspection unchecked
+ final Bindable<T> bindable = preparedResult.getBindable(cursorFactory);
+ return new CalciteSignature<>(
+ query.sql,
+ parameters,
+ preparingStmt.internalParameters,
+ jdbcType,
+ columns,
+ cursorFactory,
+ preparedResult instanceof Prepare.PreparedResultImpl
+ ? ((Prepare.PreparedResultImpl) preparedResult).collations
+ : ImmutableList.<RelCollation>of(),
+ maxRowCount,
+ bindable,
+ statementType);
+ }
+
+ private SqlValidator createSqlValidator(Context context,
+ CalciteCatalogReader catalogReader) {
+ final SqlOperatorTable opTab0 =
+ context.config().fun(SqlOperatorTable.class,
+ SqlStdOperatorTable.instance());
+ final SqlOperatorTable opTab =
+ ChainedSqlOperatorTable.of(opTab0, catalogReader);
+ final JavaTypeFactory typeFactory = context.getTypeFactory();
+ final SqlConformance conformance = context.config().conformance();
+ return new CalciteSqlValidator(opTab, catalogReader, typeFactory,
+ conformance);
+ }
+
+ private List<ColumnMetaData> getColumnMetaDataList(
+ JavaTypeFactory typeFactory, RelDataType x, RelDataType jdbcType,
+ List<List<String>> originList) {
+ final List<ColumnMetaData> columns = new ArrayList<>();
+ for (Ord<RelDataTypeField> pair : Ord.zip(jdbcType.getFieldList())) {
+ final RelDataTypeField field = pair.e;
+ final RelDataType type = field.getType();
+ final RelDataType fieldType =
+ x.isStruct() ? x.getFieldList().get(pair.i).getType() : type;
+ columns.add(
+ metaData(typeFactory, columns.size(), field.getName(), type,
+ fieldType, originList.get(pair.i)));
+ }
+ return columns;
+ }
+
+ private ColumnMetaData metaData(JavaTypeFactory typeFactory, int ordinal,
+ String fieldName, RelDataType type, RelDataType fieldType,
+ List<String> origins) {
+ final ColumnMetaData.AvaticaType avaticaType =
+ avaticaType(typeFactory, type, fieldType);
+ return new ColumnMetaData(
+ ordinal,
+ false,
+ true,
+ false,
+ false,
+ type.isNullable()
+ ? DatabaseMetaData.columnNullable
+ : DatabaseMetaData.columnNoNulls,
+ true,
+ type.getPrecision(),
+ fieldName,
+ origin(origins, 0),
+ origin(origins, 2),
+ getPrecision(type),
+ getScale(type),
+ origin(origins, 1),
+ null,
+ avaticaType,
+ true,
+ false,
+ false,
+ avaticaType.columnClassName());
+ }
+
+ private ColumnMetaData.AvaticaType avaticaType(JavaTypeFactory typeFactory,
+ RelDataType type, RelDataType fieldType) {
+ final String typeName = getTypeName(type);
+ if (type.getComponentType() != null) {
+ final ColumnMetaData.AvaticaType componentType =
+ avaticaType(typeFactory, type.getComponentType(), null);
+ final Type clazz = typeFactory.getJavaClass(type.getComponentType());
+ final ColumnMetaData.Rep rep = ColumnMetaData.Rep.of(clazz);
+ assert rep != null;
+ return ColumnMetaData.array(componentType, typeName, rep);
+ } else {
+ final int typeOrdinal = getTypeOrdinal(type);
+ switch (typeOrdinal) {
+ case Types.STRUCT:
+ final List<ColumnMetaData> columns = new ArrayList<>();
+ for (RelDataTypeField field : type.getFieldList()) {
+ columns.add(
+ metaData(typeFactory, field.getIndex(), field.getName(),
+ field.getType(), null, null));
+ }
+ return ColumnMetaData.struct(columns);
+ default:
+ final Type clazz =
+ typeFactory.getJavaClass(Util.first(fieldType, type));
+ final ColumnMetaData.Rep rep = ColumnMetaData.Rep.of(clazz);
+ assert rep != null;
+ return ColumnMetaData.scalar(typeOrdinal, typeName, rep);
+ }
+ }
+ }
+
+ private static String origin(List<String> origins, int offsetFromEnd) {
+ return origins == null || offsetFromEnd >= origins.size()
+ ? null
+ : origins.get(origins.size() - 1 - offsetFromEnd);
+ }
+
+ private int getTypeOrdinal(RelDataType type) {
+ return type.getSqlTypeName().getJdbcOrdinal();
+ }
+
+ private static String getClassName(RelDataType type) {
+ return null;
+ }
+
+ private static int getScale(RelDataType type) {
+ return type.getScale() == RelDataType.SCALE_NOT_SPECIFIED
+ ? 0
+ : type.getScale();
+ }
+
+ private static int getPrecision(RelDataType type) {
+ return type.getPrecision() == RelDataType.PRECISION_NOT_SPECIFIED
+ ? 0
+ : type.getPrecision();
+ }
+
+ /** Returns the type name in string form. Does not include precision, scale
+ * or whether nulls are allowed. Example: "DECIMAL" not "DECIMAL(7, 2)";
+ * "INTEGER" not "JavaType(int)". */
+ private static String getTypeName(RelDataType type) {
+ final SqlTypeName sqlTypeName = type.getSqlTypeName();
+ switch (sqlTypeName) {
+ case ARRAY:
+ case MULTISET:
+ case MAP:
+ case ROW:
+ return type.toString(); // e.g. "INTEGER ARRAY"
+ case INTERVAL_YEAR_MONTH:
+ return "INTERVAL_YEAR_TO_MONTH";
+ case INTERVAL_DAY_HOUR:
+ return "INTERVAL_DAY_TO_HOUR";
+ case INTERVAL_DAY_MINUTE:
+ return "INTERVAL_DAY_TO_MINUTE";
+ case INTERVAL_DAY_SECOND:
+ return "INTERVAL_DAY_TO_SECOND";
+ case INTERVAL_HOUR_MINUTE:
+ return "INTERVAL_HOUR_TO_MINUTE";
+ case INTERVAL_HOUR_SECOND:
+ return "INTERVAL_HOUR_TO_SECOND";
+ case INTERVAL_MINUTE_SECOND:
+ return "INTERVAL_MINUTE_TO_SECOND";
+ default:
+ return sqlTypeName.getName(); // e.g. "DECIMAL", "INTERVAL_YEAR_MONTH"
+ }
+ }
+
+ protected void populateMaterializations(Context context,
+ RelOptPlanner planner, Prepare.Materialization materialization) {
+ // REVIEW: initialize queryRel and tableRel inside MaterializationService,
+ // not here?
+ try {
+ final CalciteSchema schema = materialization.materializedTable.schema;
+ CalciteCatalogReader catalogReader =
+ new CalciteCatalogReader(
+ schema.root(),
+ context.config().caseSensitive(),
+ materialization.viewSchemaPath,
+ context.getTypeFactory());
+ final CalciteMaterializer materializer =
+ new CalciteMaterializer(this, context, catalogReader, schema, planner,
+ createConvertletTable());
+ materializer.populate(materialization);
+ } catch (Exception e) {
+ throw new RuntimeException("While populating materialization "
+ + materialization.materializedTable.path(), e);
+ }
+ }
+
+ private static RelDataType makeStruct(
+ RelDataTypeFactory typeFactory,
+ RelDataType type) {
+ if (type.isStruct()) {
+ return type;
+ }
+ return typeFactory.builder().add("$0", type).build();
+ }
+
+ /** Executes a prepare action. */
+ public <R> R perform(CalciteServerStatement statement,
+ Frameworks.PrepareAction<R> action) {
+ final CalcitePrepare.Context prepareContext =
+ statement.createPrepareContext();
+ final JavaTypeFactory typeFactory = prepareContext.getTypeFactory();
+ final CalciteSchema schema =
+ action.getConfig().getDefaultSchema() != null
+ ? CalciteSchema.from(action.getConfig().getDefaultSchema())
+ : prepareContext.getRootSchema();
+ CalciteCatalogReader catalogReader =
+ new CalciteCatalogReader(schema.root(),
+ prepareContext.config().caseSensitive(),
+ schema.path(null),
+ typeFactory);
+ final RexBuilder rexBuilder = new RexBuilder(typeFactory);
+ final RelOptPlanner planner =
+ createPlanner(prepareContext,
+ action.getConfig().getContext(),
+ action.getConfig().getCostFactory());
+ final RelOptCluster cluster = createCluster(planner, rexBuilder);
+ return action.apply(cluster, catalogReader,
+ prepareContext.getRootSchema().plus(), statement);
+ }
+
+ /** Holds state for the process of preparing a SQL statement. */
+ static class CalcitePreparingStmt extends Prepare
+ implements RelOptTable.ViewExpander {
+ protected final RelOptPlanner planner;
+ protected final RexBuilder rexBuilder;
+ protected final CalcitePrepareImpl prepare;
+ protected final CalciteSchema schema;
+ protected final RelDataTypeFactory typeFactory;
+ protected final SqlRexConvertletTable convertletTable;
+ private final EnumerableRel.Prefer prefer;
+ private final Map<String, Object> internalParameters =
+ Maps.newLinkedHashMap();
+ private int expansionDepth;
+ private SqlValidator sqlValidator;
+
+ public CalcitePreparingStmt(CalcitePrepareImpl prepare,
+ Context context,
+ CatalogReader catalogReader,
+ RelDataTypeFactory typeFactory,
+ CalciteSchema schema,
+ EnumerableRel.Prefer prefer,
+ RelOptPlanner planner,
+ Convention resultConvention,
+ SqlRexConvertletTable convertletTable) {
+ super(context, catalogReader, resultConvention);
+ this.prepare = prepare;
+ this.schema = schema;
+ this.prefer = prefer;
+ this.planner = planner;
+ this.typeFactory = typeFactory;
+ this.convertletTable = convertletTable;
+ this.rexBuilder = new RexBuilder(typeFactory);
+ }
+
+ @Override protected void init(Class runtimeContextClass) {
+ }
+
+ public PreparedResult prepareQueryable(
+ final Queryable queryable,
+ RelDataType resultType) {
+ return prepare_(
+ new Supplier<RelNode>() {
+ public RelNode get() {
+ final RelOptCluster cluster =
+ prepare.createCluster(planner, rexBuilder);
+ return new LixToRelTranslator(cluster, CalcitePreparingStmt.this)
+ .translate(queryable);
+ }
+ }, resultType);
+ }
+
+ public PreparedResult prepareRel(final RelNode rel) {
+ return prepare_(
+ new Supplier<RelNode>() {
+ public RelNode get() {
+ return rel;
+ }
+ }, rel.getRowType());
+ }
+
+ private PreparedResult prepare_(Supplier<RelNode> fn,
+ RelDataType resultType) {
+ queryString = null;
+ Class runtimeContextClass = Object.class;
+ init(runtimeContextClass);
+
+ final RelNode rel = fn.get();
+ final RelDataType rowType = rel.getRowType();
+ final List<Pair<Integer, String>> fields =
+ Pair.zip(ImmutableIntList.identity(rowType.getFieldCount()),
+ rowType.getFieldNames());
+ final RelCollation collation =
+ rel instanceof Sort
+ ? ((Sort) rel).collation
+ : RelCollations.EMPTY;
+ RelRoot root = new RelRoot(rel, resultType, SqlKind.SELECT, fields,
+ collation);
+
+ if (timingTracer != null) {
+ timingTracer.traceTime("end sql2rel");
+ }
+
+ final RelDataType jdbcType =
+ makeStruct(rexBuilder.getTypeFactory(), resultType);
+ fieldOrigins = Collections.nCopies(jdbcType.getFieldCount(), null);
+ parameterRowType = rexBuilder.getTypeFactory().builder().build();
+
+ // Structured type flattening, view expansion, and plugging in
+ // physical storage.
+ root = root.withRel(flattenTypes(root.rel, true));
+
+ // Trim unused fields.
+ root = trimUnusedFields(root);
+
+ final List<Materialization> materializations = ImmutableList.of();
+ final List<CalciteSchema.LatticeEntry> lattices = ImmutableList.of();
+ root = optimize(root, materializations, lattices);
+
+ if (timingTracer != null) {
+ timingTracer.traceTime("end optimization");
+ }
+
+ return implement(root);
+ }
+
+ @Override protected SqlToRelConverter getSqlToRelConverter(
+ SqlValidator validator,
+ CatalogReader catalogReader,
+ SqlToRelConverter.Config config) {
+ final RelOptCluster cluster = prepare.createCluster(planner, rexBuilder);
+ SqlToRelConverter sqlToRelConverter =
+ new SqlToRelConverter(this, validator, catalogReader, cluster,
+ convertletTable, config);
+ return sqlToRelConverter;
+ }
+
+ @Override public RelNode flattenTypes(
+ RelNode rootRel,
+ boolean restructure) {
+ final SparkHandler spark = context.spark();
+ if (spark.enabled()) {
+ return spark.flattenTypes(planner, rootRel, restructure);
+ }
+ return rootRel;
+ }
+
+ @Override protected RelNode decorrelate(SqlToRelConverter sqlToRelConverter,
+ SqlNode query, RelNode rootRel) {
+ return sqlToRelConverter.decorrelate(query, rootRel);
+ }
+
+ @Override public RelRoot expandView(RelDataType rowType, String queryString,
+ List<String> schemaPath, List<String> viewPath) {
+ expansionDepth++;
+
+ SqlParser parser = prepare.createParser(queryString);
+ SqlNode sqlNode;
+ try {
+ sqlNode = parser.parseQuery();
+ } catch (SqlParseException e) {
+ throw new RuntimeException("parse failed", e);
+ }
+ // View may have different schema path than current connection.
+ final CatalogReader catalogReader =
+ this.catalogReader.withSchemaPath(schemaPath);
+ SqlValidator validator = createSqlValidator(catalogReader);
+ SqlNode sqlNode1 = validator.validate(sqlNode);
+ final SqlToRelConverter.Config config = SqlToRelConverter.configBuilder()
+ .withTrimUnusedFields(true).build();
+ SqlToRelConverter sqlToRelConverter =
+ getSqlToRelConverter(validator, catalogReader, config);
+ RelRoot root =
+ sqlToRelConverter.convertQuery(sqlNode1, true, false);
+
+ --expansionDepth;
+ return root;
+ }
+
+ protected SqlValidator createSqlValidator(CatalogReader catalogReader) {
+ return prepare.createSqlValidator(context,
+ (CalciteCatalogReader) catalogReader);
+ }
+
+ @Override protected SqlValidator getSqlValidator() {
+ if (sqlValidator == null) {
+ sqlValidator = createSqlValidator(catalogReader);
+ }
+ return sqlValidator;
+ }
+
+ @Override protected PreparedResult createPreparedExplanation(
+ RelDataType resultType,
+ RelDataType parameterRowType,
+ RelRoot root,
+ SqlExplainFormat format,
+ SqlExplainLevel detailLevel) {
+ return new CalcitePreparedExplain(resultType, parameterRowType, root,
+ format, detailLevel);
+ }
+
+ @Override protected PreparedResult implement(RelRoot root) {
+ RelDataType resultType = root.rel.getRowType();
+ boolean isDml = root.kind.belongsTo(SqlKind.DML);
+ final Bindable bindable;
+ if (resultConvention == BindableConvention.INSTANCE) {
+ bindable = Interpreters.bindable(root.rel);
+ } else {
+ EnumerableRel enumerable = (EnumerableRel) root.rel;
+ if (!root.isRefTrivial()) {
+ final List<RexNode> projects = new ArrayList<>();
+ final RexBuilder rexBuilder = enumerable.getCluster().getRexBuilder();
+ for (int field : Pair.left(root.fields)) {
+ projects.add(rexBuilder.makeInputRef(enumerable, field));
+ }
+ RexProgram program = RexProgram.create(enumerable.getRowType(),
+ projects, null, root.validatedRowType, rexBuilder);
+ enumerable = EnumerableCalc.create(enumerable, program);
+ }
+
+ try {
+ CatalogReader.THREAD_LOCAL.set(catalogReader);
+ bindable = EnumerableInterpretable.toBindable(internalParameters,
+ context.spark(), enumerable, prefer);
+ } finally {
+ CatalogReader.THREAD_LOCAL.remove();
+ }
+ }
+
+ if (timingTracer != null) {
+ timingTracer.traceTime("end codegen");
+ }
+
+ if (timingTracer != null) {
+ timingTracer.traceTime("end compilation");
+ }
+
+ return new PreparedResultImpl(
+ resultType,
+ parameterRowType,
+ fieldOrigins,
+ root.collation.getFieldCollations().isEmpty()
+ ? ImmutableList.<RelCollation>of()
+ : ImmutableList.of(root.collation),
+ root.rel,
+ mapTableModOp(isDml, root.kind),
+ isDml) {
+ public String getCode() {
+ throw new UnsupportedOperationException();
+ }
+
+ public Bindable getBindable(Meta.CursorFactory cursorFactory) {
+ return bindable;
+ }
+
+ public Type getElementType() {
+ return ((Typed) bindable).getElementType();
+ }
+ };
+ }
+
+ @Override protected List<Materialization> getMaterializations() {
+ final List<Prepare.Materialization> materializations =
+ context.config().materializationsEnabled()
+ ? MaterializationService.instance().query(schema)
+ : ImmutableList.<Prepare.Materialization>of();
+ for (Prepare.Materialization materialization : materializations) {
+ prepare.populateMaterializations(context, planner, materialization);
+ }
+ return materializations;
+ }
+
+ @Override protected List<LatticeEntry> getLattices() {
+ return Schemas.getLatticeEntries(schema);
+ }
+ }
+
+ /** An {@code EXPLAIN} statement, prepared and ready to execute. */
+ private static class CalcitePreparedExplain extends Prepare.PreparedExplain {
+ public CalcitePreparedExplain(
+ RelDataType resultType,
+ RelDataType parameterRowType,
+ RelRoot root,
+ SqlExplainFormat format,
+ SqlExplainLevel detailLevel) {
+ super(resultType, parameterRowType, root, format, detailLevel);
+ }
+
+ public Bindable getBindable(final Meta.CursorFactory cursorFactory) {
+ final String explanation = getCode();
+ return new Bindable() {
+ public Enumerable bind(DataContext dataContext) {
+ switch (cursorFactory.style) {
+ case ARRAY:
+ return Linq4j.singletonEnumerable(new String[] {explanation});
+ case OBJECT:
+ default:
+ return Linq4j.singletonEnumerable(explanation);
+ }
+ }
+ };
+ }
+ }
+
+ /** Translator from Java AST to {@link RexNode}. */
+ interface ScalarTranslator {
+ RexNode toRex(BlockStatement statement);
+ List<RexNode> toRexList(BlockStatement statement);
+ RexNode toRex(Expression expression);
+ ScalarTranslator bind(List<ParameterExpression> parameterList,
+ List<RexNode> values);
+ }
+
+ /** Basic translator. */
+ static class EmptyScalarTranslator implements ScalarTranslator {
+ private final RexBuilder rexBuilder;
+
+ public EmptyScalarTranslator(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ }
+
+ public static ScalarTranslator empty(RexBuilder builder) {
+ return new EmptyScalarTranslator(builder);
+ }
+
+ public List<RexNode> toRexList(BlockStatement statement) {
+ final List<Expression> simpleList = simpleList(statement);
+ final List<RexNode> list = new ArrayList<>();
+ for (Expression expression1 : simpleList) {
+ list.add(toRex(expression1));
+ }
+ return list;
+ }
+
+ public RexNode toRex(BlockStatement statement) {
+ return toRex(Blocks.simple(statement));
+ }
+
+ private static List<Expression> simpleList(BlockStatement statement) {
+ Expression simple = Blocks.simple(statement);
+ if (simple instanceof NewExpression) {
+ NewExpression newExpression = (NewExpression) simple;
+ return newExpression.arguments;
+ } else {
+ return Collections.singletonList(simple);
+ }
+ }
+
+ public RexNode toRex(Expression expression) {
+ switch (expression.getNodeType()) {
+ case MemberAccess:
+ // Case-sensitive name match because name was previously resolved.
+ return rexBuilder.makeFieldAccess(
+ toRex(
+ ((MemberExpression) expression).expression),
+ ((MemberExpression) expression).field.getName(),
+ true);
+ case GreaterThan:
+ return binary(expression, SqlStdOperatorTable.GREATER_THAN);
+ case LessThan:
+ return binary(expression, SqlStdOperatorTable.LESS_THAN);
+ case Parameter:
+ return parameter((ParameterExpression) expression);
+ case Call:
+ MethodCallExpression call = (MethodCallExpression) expression;
+ SqlOperator operator =
+ RexToLixTranslator.JAVA_TO_SQL_METHOD_MAP.get(call.method);
+ if (operator != null) {
+ return rexBuilder.makeCall(
+ type(call),
+ operator,
+ toRex(
+ Expressions.<Expression>list()
+ .appendIfNotNull(call.targetExpression)
+ .appendAll(call.expressions)));
+ }
+ throw new RuntimeException(
+ "Could translate call to method " + call.method);
+ case Constant:
+ final ConstantExpression constant =
+ (ConstantExpression) expression;
+ Object value = constant.value;
+ if (value instanceof Number) {
+ Number number = (Number) value;
+ if (value instanceof Double || value instanceof Float) {
+ return rexBuilder.makeApproxLiteral(
+ BigDecimal.valueOf(number.doubleValue()));
+ } else if (value instanceof BigDecimal) {
+ return rexBuilder.makeExactLiteral((BigDecimal) value);
+ } else {
+ return rexBuilder.makeExactLiteral(
+ BigDecimal.valueOf(number.longValue()));
+ }
+ } else if (value instanceof Boolean) {
+ return rexBuilder.makeLiteral((Boolean) value);
+ } else {
+ return rexBuilder.makeLiteral(constant.toString());
+ }
+ default:
+ throw new UnsupportedOperationException(
+ "unknown expression type " + expression.getNodeType() + " "
+ + expression);
+ }
+ }
+
+ private RexNode binary(Expression expression, SqlBinaryOperator op) {
+ BinaryExpression call = (BinaryExpression) expression;
+ return rexBuilder.makeCall(type(call), op,
+ toRex(ImmutableList.of(call.expression0, call.expression1)));
+ }
+
+ private List<RexNode> toRex(List<Expression> expressions) {
+ final List<RexNode> list = new ArrayList<>();
+ for (Expression expression : expressions) {
+ list.add(toRex(expression));
+ }
+ return list;
+ }
+
+ protected RelDataType type(Expression expression) {
+ final Type type = expression.getType();
+ return ((JavaTypeFactory) rexBuilder.getTypeFactory()).createType(type);
+ }
+
+ public ScalarTranslator bind(
+ List<ParameterExpression> parameterList, List<RexNode> values) {
+ return new LambdaScalarTranslator(
+ rexBuilder, parameterList, values);
+ }
+
+ public RexNode parameter(ParameterExpression param) {
+ throw new RuntimeException("unknown parameter " + param);
+ }
+ }
+
+ /** Translator that looks for parameters. */
+ private static class LambdaScalarTranslator extends EmptyScalarTranslator {
+ private final List<ParameterExpression> parameterList;
+ private final List<RexNode> values;
+
+ public LambdaScalarTranslator(
+ RexBuilder rexBuilder,
+ List<ParameterExpression> parameterList,
+ List<RexNode> values) {
+ super(rexBuilder);
+ this.parameterList = parameterList;
+ this.values = values;
+ }
+
+ public RexNode parameter(ParameterExpression param) {
+ int i = parameterList.indexOf(param);
+ if (i >= 0) {
+ return values.get(i);
+ }
+ throw new RuntimeException("unknown parameter " + param);
+ }
+ }
+}
+
+// End CalcitePrepareImpl.java
http://git-wip-us.apache.org/repos/asf/kylin/blob/56e9fb9e/atopcalcite/src/main/java/org/apache/calcite/prepare/OnlyPrepareEarlyAbortException.java
----------------------------------------------------------------------
diff --git a/atopcalcite/src/main/java/org/apache/calcite/prepare/OnlyPrepareEarlyAbortException.java b/atopcalcite/src/main/java/org/apache/calcite/prepare/OnlyPrepareEarlyAbortException.java
new file mode 100644
index 0000000..8493484
--- /dev/null
+++ b/atopcalcite/src/main/java/org/apache/calcite/prepare/OnlyPrepareEarlyAbortException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.calcite.prepare;
+
+import org.apache.calcite.jdbc.CalcitePrepare;
+
+public class OnlyPrepareEarlyAbortException extends RuntimeException {
+
+ private CalcitePrepare.Context context;
+ private org.apache.calcite.jdbc.CalcitePrepare.ParseResult preparedResult;
+
+ public OnlyPrepareEarlyAbortException(CalcitePrepare.Context context,
+ org.apache.calcite.jdbc.CalcitePrepare.ParseResult preparedResult) {
+ this.context = context;
+ this.preparedResult = preparedResult;
+ }
+
+ public CalcitePrepare.Context getContext() {
+ return context;
+ }
+
+ public CalcitePrepare.ParseResult getPreparedResult() {
+ return preparedResult;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/56e9fb9e/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 0da92c7..35dcee4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -22,10 +22,13 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import javax.servlet.http.HttpServletResponse;
+import com.google.common.collect.Maps;
import org.apache.commons.io.IOUtils;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -76,6 +79,10 @@ public class QueryController extends BasicController {
@RequestMapping(value = "/query/prestate", method = RequestMethod.POST, produces = "application/json")
@ResponseBody
public SQLResponse prepareQuery(@RequestBody PrepareSqlRequest sqlRequest) {
+ Map<String, String> toggles = Maps.newHashMap();
+ toggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
+ BackdoorToggles.addToggles(toggles);
+
return queryService.doQueryWithCache(sqlRequest);
}
@@ -106,7 +113,7 @@ public class QueryController extends BasicController {
@ResponseBody
public void downloadQueryResult(@PathVariable String format, SQLRequest sqlRequest, HttpServletResponse response) {
SQLResponse result = queryService.doQueryWithCache(sqlRequest);
- response.setContentType("text/" + format + ";charset=ansi");
+ response.setContentType("text/" + format + ";charset=utf-8");
response.setHeader("Content-Disposition", "attachment; filename=\"result." + format + "\"");
ICsvListWriter csvWriter = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/56e9fb9e/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index b0ed171..1b118a3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -47,6 +47,12 @@ import javax.annotation.PostConstruct;
import javax.sql.DataSource;
import org.apache.calcite.avatica.ColumnMetaData.Rep;
+import org.apache.calcite.config.CalciteConnectionConfig;
+import org.apache.calcite.jdbc.CalcitePrepare;
+import org.apache.calcite.prepare.CalcitePrepareImpl;
+import org.apache.calcite.prepare.OnlyPrepareEarlyAbortException;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.sql.type.BasicSqlType;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.kylin.common.KylinConfig;
@@ -117,12 +123,10 @@ import net.sf.ehcache.Element;
@Component("queryService")
public class QueryService extends BasicService {
- private static final Logger logger = LoggerFactory.getLogger(QueryService.class);
-
public static final String SUCCESS_QUERY_CACHE = "StorageCache";
public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache";
public static final String QUERY_STORE_PATH_PREFIX = "/query/";
-
+ private static final Logger logger = LoggerFactory.getLogger(QueryService.class);
final BadQueryDetector badQueryDetector = new BadQueryDetector();
final ResourceStore queryStore;
@@ -140,16 +144,27 @@ public class QueryService extends BasicService {
@Autowired
private AclUtil aclUtil;
- @PostConstruct
- public void init() throws IOException {
- Preconditions.checkNotNull(cacheManager, "cacheManager is not injected yet");
- }
-
public QueryService() {
queryStore = ResourceStore.getStore(getConfig());
badQueryDetector.start();
}
+ protected static void close(ResultSet resultSet, Statement stat, Connection conn) {
+ OLAPContext.clearParameter();
+ DBUtils.closeQuietly(resultSet);
+ DBUtils.closeQuietly(stat);
+ DBUtils.closeQuietly(conn);
+ }
+
+ private static String getQueryKeyById(String creator) {
+ return QUERY_STORE_PATH_PREFIX + creator;
+ }
+
+ @PostConstruct
+ public void init() throws IOException {
+ Preconditions.checkNotNull(cacheManager, "cacheManager is not injected yet");
+ }
+
public List<TableMeta> getMetadata(String project) throws SQLException {
return getMetadata(getCubeManager(), project, true);
}
@@ -535,6 +550,7 @@ public class QueryService extends BasicService {
return getMetadataV2(getCubeManager(), project, true);
}
+ @SuppressWarnings("checkstyle:methodlength")
protected List<TableMetaWithType> getMetadataV2(CubeManager cubeMgr, String project, boolean cubedOnly) throws SQLException, IOException {
//Message msg = MsgPicker.getMsg();
@@ -695,21 +711,15 @@ public class QueryService extends BasicService {
try {
conn = cacheService.getOLAPDataSource(sqlRequest.getProject()).getConnection();
- if (sqlRequest instanceof PrepareSqlRequest) {
- PreparedStatement preparedState = conn.prepareStatement(correctedSql);
- processStatementAttr(preparedState, sqlRequest);
-
- for (int i = 0; i < ((PrepareSqlRequest) sqlRequest).getParams().length; i++) {
- setParam(preparedState, i + 1, ((PrepareSqlRequest) sqlRequest).getParams()[i]);
- }
-
- resultSet = preparedState.executeQuery();
- } else {
- stat = conn.createStatement();
- processStatementAttr(stat, sqlRequest);
- resultSet = stat.executeQuery(correctedSql);
+ // special case for prepare query.
+ if (BackdoorToggles.getPrepareOnly()) {
+ return getPrepareOnlySqlResponse(correctedSql, conn, isAdHoc, results, columnMetas);
}
+ stat = conn.createStatement();
+ processStatementAttr(stat, sqlRequest);
+ resultSet = stat.executeQuery(correctedSql);
+
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();
@@ -733,6 +743,50 @@ public class QueryService extends BasicService {
close(resultSet, stat, conn);
}
+ return getSqlResponse(isAdHoc, results, columnMetas);
+ }
+
+ private SQLResponse getPrepareOnlySqlResponse(String correctedSql, Connection conn,
+ Boolean isAdHoc, List<List<String>> results, List<SelectedColumnMeta> columnMetas)
+ throws SQLException {
+
+ CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(true);
+
+ try {
+ conn.prepareStatement(correctedSql);
+ throw new IllegalStateException("Should have thrown OnlyPrepareEarlyAbortException");
+ } catch (Exception e) {
+ Throwable rootCause = ExceptionUtils.getRootCause(e);
+ if (rootCause != null && rootCause instanceof OnlyPrepareEarlyAbortException) {
+ OnlyPrepareEarlyAbortException abortException = (OnlyPrepareEarlyAbortException) rootCause;
+ CalcitePrepare.Context context = abortException.getContext();
+ CalcitePrepare.ParseResult preparedResult = abortException.getPreparedResult();
+ List<RelDataTypeField> fieldList = preparedResult.rowType.getFieldList();
+
+ CalciteConnectionConfig config = context.config();
+
+ // Fill in selected column meta
+ for (int i = 0; i < fieldList.size(); ++i) {
+
+ RelDataTypeField field = fieldList.get(i);
+ String columnName = field.getKey();
+ BasicSqlType basicSqlType = (BasicSqlType) field.getValue();
+
+ columnMetas.add(new SelectedColumnMeta(false, config.caseSensitive(), false, false, basicSqlType.isNullable() ? 1 : 0, true, basicSqlType.getPrecision(), columnName, columnName, null, null, null, basicSqlType.getPrecision(), basicSqlType.getScale(), basicSqlType.getSqlTypeName().getJdbcOrdinal(), basicSqlType.getSqlTypeName().getName(), true, false, false));
+ }
+
+ } else {
+ throw e;
+ }
+ } finally {
+ CalcitePrepareImpl.KYLIN_ONLY_PREPARE.set(false);
+ }
+
+ return getSqlResponse(isAdHoc, results, columnMetas);
+ }
+
+ private SQLResponse getSqlResponse(Boolean isAdHoc, List<List<String>> results, List<SelectedColumnMeta> columnMetas) {
+
boolean isPartialResult = false;
StringBuilder cubeSb = new StringBuilder();
StringBuilder logSb = new StringBuilder("Processed rows for each storageContext: ");
@@ -753,7 +807,6 @@ public class QueryService extends BasicService {
SQLResponse response = new SQLResponse(columnMetas, results, cubeSb.toString(), 0, false, null, isPartialResult, isAdHoc);
response.setTotalScanCount(QueryContext.current().getScannedRows());
response.setTotalScanBytes(QueryContext.current().getScannedBytes());
-
return response;
}
@@ -839,21 +892,10 @@ public class QueryService extends BasicService {
}
}
- protected static void close(ResultSet resultSet, Statement stat, Connection conn) {
- OLAPContext.clearParameter();
- DBUtils.closeQuietly(resultSet);
- DBUtils.closeQuietly(stat);
- DBUtils.closeQuietly(conn);
- }
-
public void setCacheManager(CacheManager cacheManager) {
this.cacheManager = cacheManager;
}
- private static String getQueryKeyById(String creator) {
- return QUERY_STORE_PATH_PREFIX + creator;
- }
-
private static class QueryRecordSerializer implements Serializer<QueryRecord> {
private static final QueryRecordSerializer serializer = new QueryRecordSerializer();
[27/50] kylin git commit: minor, user and acl jason serializer
Posted by li...@apache.org.
minor, user and acl jason serializer
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cfac81bf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cfac81bf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cfac81bf
Branch: refs/heads/master
Commit: cfac81bffc22e1fb320c9fd6efe8c4e4ffd219a5
Parents: bf87169
Author: Roger Shi <ro...@hotmail.com>
Authored: Wed Jun 21 15:45:28 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Wed Jun 21 16:27:16 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/service/AclService.java | 45 +++++---------------
.../rest/service/AclTableMigrationTool.java | 7 +--
.../apache/kylin/rest/service/UserService.java | 37 +++-------------
3 files changed, 21 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfac81bf/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
index 6292c00..02b8e9f 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java
@@ -18,8 +18,6 @@
package org.apache.kylin.rest.service;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
@@ -29,10 +27,10 @@ import java.util.List;
import java.util.Map;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.msg.Message;
@@ -79,6 +77,8 @@ public class AclService implements MutableAclService {
public static final String DIR_PREFIX = "/acl/";
+ public static final Serializer<AclRecord> SERIALIZER = new JsonSerializer<>(AclRecord.class);
+
@Autowired
protected PermissionGrantingStrategy permissionGrantingStrategy;
@@ -107,7 +107,8 @@ public class AclService implements MutableAclService {
public List<ObjectIdentity> findChildren(ObjectIdentity parentIdentity) {
List<ObjectIdentity> oids = new ArrayList<ObjectIdentity>();
try {
- List<AclRecord> allAclRecords = aclStore.getAllResources(String.valueOf(DIR_PREFIX), AclRecord.class, AclRecordSerializer.getInstance());
+ List<AclRecord> allAclRecords = aclStore.getAllResources(String.valueOf(DIR_PREFIX), AclRecord.class,
+ SERIALIZER);
for (AclRecord record : allAclRecords) {
DomainObjectInfo parent = record.getParentDomainObjectInfo();
if (parent != null && parent.getId().equals(String.valueOf(parentIdentity.getIdentifier()))) {
@@ -148,7 +149,8 @@ public class AclService implements MutableAclService {
Map<ObjectIdentity, Acl> aclMaps = new HashMap<ObjectIdentity, Acl>();
try {
for (ObjectIdentity oid : oids) {
- AclRecord record = aclStore.getResource(getQueryKeyById(String.valueOf(oid.getIdentifier())), AclRecord.class, AclRecordSerializer.getInstance());
+ AclRecord record = aclStore.getResource(getQueryKeyById(String.valueOf(oid.getIdentifier())),
+ AclRecord.class, SERIALIZER);
if (record != null) {
SidInfo owner = record.getOwnerInfo();
Sid ownerSid = (null == owner) ? null : (owner.isPrincipal() ? new PrincipalSid(owner.getSid()) : new GrantedAuthoritySid(owner.getSid()));
@@ -191,7 +193,8 @@ public class AclService implements MutableAclService {
PrincipalSid sid = new PrincipalSid(auth);
try {
AclRecord record = new AclRecord(new DomainObjectInfo(objectIdentity), null, new SidInfo(sid), true, null);
- aclStore.putResource(getQueryKeyById(String.valueOf(objectIdentity.getIdentifier())), record, 0, AclRecordSerializer.getInstance());
+ aclStore.putResource(getQueryKeyById(String.valueOf(objectIdentity.getIdentifier())), record, 0,
+ SERIALIZER);
logger.debug("ACL of " + objectIdentity + " created successfully.");
} catch (IOException e) {
throw new InternalErrorException(e);
@@ -228,7 +231,7 @@ public class AclService implements MutableAclService {
try {
String id = getQueryKeyById(String.valueOf(mutableAcl.getObjectIdentity().getIdentifier()));
- AclRecord record = aclStore.getResource(id, AclRecord.class, AclRecordSerializer.getInstance());
+ AclRecord record = aclStore.getResource(id, AclRecord.class, SERIALIZER);
aclStore.deleteResource(id);
if (mutableAcl.getParentAcl() != null) {
record.setParentDomainObjectInfo(new DomainObjectInfo(mutableAcl.getParentAcl().getObjectIdentity()));
@@ -249,7 +252,7 @@ public class AclService implements MutableAclService {
AceInfo aceInfo = new AceInfo(ace);
allAceInfo.put(String.valueOf(aceInfo.getSidInfo().getSid()), aceInfo);
}
- aclStore.putResource(id, record, 0, AclRecordSerializer.getInstance());
+ aclStore.putResource(id, record, 0, SERIALIZER);
logger.debug("ACL of " + mutableAcl.getObjectIdentity() + " updated successfully.");
} catch (IOException e) {
throw new InternalErrorException(e);
@@ -307,32 +310,6 @@ public class AclService implements MutableAclService {
public static String getQueryKeyById(String id) {
return DIR_PREFIX + id;
}
-
- protected static class AclRecordSerializer implements Serializer<AclRecord> {
-
- private static final AclRecordSerializer serializer = new AclRecordSerializer();
-
- AclRecordSerializer() {
-
- }
-
- public static AclRecordSerializer getInstance() {
- return serializer;
- }
-
- @Override
- public void serialize(AclRecord obj, DataOutputStream out) throws IOException {
- String jsonStr = JsonUtil.writeValueAsString(obj);
- out.writeUTF(jsonStr);
- }
-
- @Override
- public AclRecord deserialize(DataInputStream in) throws IOException {
- String jsonStr = in.readUTF();
- return JsonUtil.readValue(jsonStr, AclRecord.class);
- }
- }
-
}
@SuppressWarnings("serial")
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfac81bf/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
index 428c556..fc50410 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
@@ -128,7 +128,7 @@ public class AclTableMigrationTool {
record.setEntriesInheriting(getInheriting(result));
record.setAllAceInfo(getAllAceInfo(result));
store.deleteResource(AclService.getQueryKeyById(object.getId()));
- store.putResource(AclService.getQueryKeyById(object.getId()), record, 0, AclService.AclRecordSerializer.getInstance());
+ store.putResource(AclService.getQueryKeyById(object.getId()), record, 0, AclService.SERIALIZER);
result = rs.next();
}
}
@@ -147,7 +147,8 @@ public class AclTableMigrationTool {
User user = hbaseRowToUser(result);
UserInfo userInfo = convert(user);
store.deleteResource(UserService.getId(userInfo.getUsername()));
- store.putResource(UserService.getId(userInfo.getUsername()), userInfo, 0, UserService.UserInfoSerializer.getInstance());
+ store.putResource(UserService.getId(userInfo.getUsername()), userInfo, 0,
+ UserService.SERIALIZER);
result = rs.next();
}
}
@@ -190,7 +191,7 @@ public class AclTableMigrationTool {
private DomainObjectInfo getDomainObjectInfoFromRs(Result result) {
String type = String.valueOf(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY), Bytes.toBytes(AclConstant.ACL_INFO_FAMILY_TYPE_COLUMN)));
- String id = String.valueOf(result.getRow());
+ String id = new String(result.getRow());
DomainObjectInfo newInfo = new DomainObjectInfo();
newInfo.setId(id);
newInfo.setType(type);
http://git-wip-us.apache.org/repos/asf/kylin/blob/cfac81bf/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 9adfcb8..e803040 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -18,8 +18,6 @@
package org.apache.kylin.rest.service;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -27,9 +25,9 @@ import java.util.List;
import javax.annotation.PostConstruct;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
@@ -51,6 +49,8 @@ public class UserService implements UserDetailsManager {
public static final String DIR_PREFIX = "/user/";
+ public static final Serializer<UserInfo> SERIALIZER = new JsonSerializer<>(UserInfo.class);
+
protected ResourceStore aclStore;
@PostConstruct
@@ -70,7 +70,7 @@ public class UserService implements UserDetailsManager {
try {
deleteUser(user.getUsername());
String id = getId(user.getUsername());
- aclStore.putResource(id, new UserInfo(user), 0, UserInfoSerializer.getInstance());
+ aclStore.putResource(id, new UserInfo(user), 0, SERIALIZER);
logger.debug("update user : {}", user.getUsername());
} catch (IOException e) {
throw new InternalErrorException(e);
@@ -107,7 +107,7 @@ public class UserService implements UserDetailsManager {
public UserDetails loadUserByUsername(String userName) throws UsernameNotFoundException {
Message msg = MsgPicker.getMsg();
try {
- UserInfo userInfo = aclStore.getResource(getId(userName), UserInfo.class, UserInfoSerializer.getInstance());
+ UserInfo userInfo = aclStore.getResource(getId(userName), UserInfo.class, SERIALIZER);
if (userInfo == null) {
throw new UsernameNotFoundException(String.format(msg.getUSER_NOT_FOUND(), userName));
}
@@ -132,7 +132,7 @@ public class UserService implements UserDetailsManager {
public List<UserDetails> listUsers() throws IOException {
List<UserDetails> all = new ArrayList<UserDetails>();
- List<UserInfo> userInfos = aclStore.getAllResources(DIR_PREFIX, UserInfo.class, UserInfoSerializer.getInstance());
+ List<UserInfo> userInfos = aclStore.getAllResources(DIR_PREFIX, UserInfo.class, SERIALIZER);
for (UserInfo info : userInfos) {
all.add(wrap(info));
}
@@ -154,29 +154,4 @@ public class UserService implements UserDetailsManager {
return new User(userInfo.getUsername(), userInfo.getPassword(), authorities);
}
- public static class UserInfoSerializer implements Serializer<UserInfo> {
-
- private static final UserInfoSerializer serializer = new UserInfoSerializer();
-
- private UserInfoSerializer() {
-
- }
-
- public static UserInfoSerializer getInstance() {
- return serializer;
- }
-
- @Override
- public void serialize(UserInfo userInfo, DataOutputStream out) throws IOException {
- String json = JsonUtil.writeValueAsString(userInfo);
- out.writeUTF(json);
- }
-
- @Override
- public UserInfo deserialize(DataInputStream in) throws IOException {
- String json = in.readUTF();
- return JsonUtil.readValue(json, UserInfo.class);
- }
- }
-
}
[45/50] kylin git commit: minor,
fix manager inconsistence with thread-local KylinConfig
Posted by li...@apache.org.
minor, fix manager inconsistence with thread-local KylinConfig
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4f20ba32
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4f20ba32
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4f20ba32
Branch: refs/heads/master
Commit: 4f20ba32a8e93c05957d6da8e71b71920828a4f3
Parents: 450845c
Author: lidongsjtu <li...@apache.org>
Authored: Sun Jun 25 18:18:55 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Tue Jun 27 18:28:14 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/cube/CubeDescManager.java | 5 +++++
.../src/main/java/org/apache/kylin/cube/CubeManager.java | 5 +++++
.../main/java/org/apache/kylin/dict/DictionaryManager.java | 5 +++++
.../org/apache/kylin/job/execution/ExecutableManager.java | 5 +++++
.../main/java/org/apache/kylin/metadata/MetadataManager.java | 5 +++++
.../kylin/metadata/badquery/BadQueryHistoryManager.java | 5 +++++
.../org/apache/kylin/metadata/cachesync/Broadcaster.java | 8 ++++++++
.../java/org/apache/kylin/metadata/draft/DraftManager.java | 5 +++++
.../org/apache/kylin/metadata/project/ProjectManager.java | 5 +++++
.../kylin/metadata/realization/RealizationRegistry.java | 4 ++++
.../java/org/apache/kylin/storage/hybrid/HybridManager.java | 5 +++++
11 files changed, 57 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f20ba32/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index 4e4daf2..2d44454 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -94,6 +94,11 @@ public class CubeDescManager {
CACHE.clear();
}
+ public static void clearCache(KylinConfig kylinConfig) {
+ if (kylinConfig != null)
+ CACHE.remove(kylinConfig);
+ }
+
// ============================================================================
private KylinConfig config;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f20ba32/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 32e2316..da3abf1 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -124,6 +124,11 @@ public class CubeManager implements IRealizationProvider {
CACHE.clear();
}
+ public static void clearCache(KylinConfig kylinConfig) {
+ if (kylinConfig != null)
+ CACHE.remove(kylinConfig);
+ }
+
// ============================================================================
private KylinConfig config;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f20ba32/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 1628f4e..857ee30 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -83,6 +83,11 @@ public class DictionaryManager {
CACHE.clear();
}
+ public static void clearCache(KylinConfig kylinConfig) {
+ if (kylinConfig != null)
+ CACHE.remove(kylinConfig);
+ }
+
// ============================================================================
private KylinConfig config;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f20ba32/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 170a254..f16cfde 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -83,6 +83,11 @@ public class ExecutableManager {
CACHE.clear();
}
+ public static void clearCache(KylinConfig kylinConfig) {
+ if (kylinConfig != null)
+ CACHE.remove(kylinConfig);
+ }
+
private static ExecutablePO parse(AbstractExecutable executable) {
ExecutablePO result = new ExecutablePO();
result.setName(executable.getName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f20ba32/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 2a894b9..96ae294 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -103,6 +103,11 @@ public class MetadataManager {
CACHE.clear();
}
+ public static void clearCache(KylinConfig kylinConfig) {
+ if (kylinConfig != null)
+ CACHE.remove(kylinConfig);
+ }
+
// ============================================================================
private KylinConfig config;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f20ba32/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
index c7eb133..d7cd425 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
@@ -72,6 +72,11 @@ public class BadQueryHistoryManager {
CACHE.clear();
}
+ public static void clearCache(KylinConfig kylinConfig) {
+ if (kylinConfig != null)
+ CACHE.remove(kylinConfig);
+ }
+
private ResourceStore getStore() {
return ResourceStore.getStore(this.kylinConfig);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f20ba32/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index c9e1130..4b0ef57 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -93,6 +93,14 @@ public class Broadcaster {
}
}
+ public static void clearCache(KylinConfig kylinConfig) {
+ if (kylinConfig != null) {
+ synchronized (CACHE) {
+ CACHE.remove(kylinConfig);
+ }
+ }
+ }
+
// ============================================================================
private KylinConfig config;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f20ba32/core-metadata/src/main/java/org/apache/kylin/metadata/draft/DraftManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/draft/DraftManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/draft/DraftManager.java
index 56ee251..bf3c3c3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/draft/DraftManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/draft/DraftManager.java
@@ -72,6 +72,11 @@ public class DraftManager {
CACHE.clear();
}
+ public static void clearCache(KylinConfig kylinConfig) {
+ if (kylinConfig != null)
+ CACHE.remove(kylinConfig);
+ }
+
// ============================================================================
private KylinConfig config;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f20ba32/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index 213b136..ea03c3c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -85,6 +85,11 @@ public class ProjectManager {
CACHE.clear();
}
+ public static void clearCache(KylinConfig kylinConfig) {
+ if (kylinConfig != null)
+ CACHE.remove(kylinConfig);
+ }
+
// ============================================================================
private KylinConfig config;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f20ba32/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
index 2d1a4a5..2ae4fa1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java
@@ -68,6 +68,10 @@ public class RealizationRegistry {
CACHE.clear();
}
+ public static void clearCache(KylinConfig kylinConfig) {
+ CACHE.remove(kylinConfig);
+ }
+
// ============================================================================
private Map<RealizationType, IRealizationProvider> providers;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4f20ba32/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
index cf40416..114f2c7 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -79,6 +79,11 @@ public class HybridManager implements IRealizationProvider {
CACHE.clear();
}
+ public static void clearCache(KylinConfig kylinConfig) {
+ if (kylinConfig != null)
+ CACHE.remove(kylinConfig);
+ }
+
// ============================================================================
private KylinConfig config;
[29/50] kylin git commit: KYLIN-2676 Allow backup operation copying
UUID
Posted by li...@apache.org.
KYLIN-2676 Allow backup operation copying UUID
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/071f3b92
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/071f3b92
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/071f3b92
Branch: refs/heads/master
Commit: 071f3b92caccf56ed70c15147da32a9ef2538bff
Parents: 57c1d5e
Author: auphyroc99 <45...@qq.com>
Authored: Thu Jun 22 17:10:58 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 22 18:05:21 2017 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/ResourceTool.java | 40 ++++++++++++++------
.../apache/kylin/tool/CubeMetaExtractor.java | 4 +-
2 files changed, 31 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/071f3b92/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 6a73e12..2571c91 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -72,10 +72,10 @@ public class ResourceTool {
list(KylinConfig.getInstanceFromEnv(), args[1]);
break;
case "download":
- copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]));
+ copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]), true);
break;
case "fetch":
- copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]), args[2]);
+ copy(KylinConfig.getInstanceFromEnv(), KylinConfig.createInstanceFromUri(args[1]), args[2], true);
break;
case "upload":
copy(KylinConfig.createInstanceFromUri(args[1]), KylinConfig.getInstanceFromEnv());
@@ -150,37 +150,54 @@ public class ResourceTool {
System.out.println("" + result);
return result;
}
+
+ public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, String path) throws IOException {
+ copy(srcConfig, dstConfig, path, false);
+ }
- public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, String path) throws IOException {
+ //Do NOT invoke this method directly, unless you want to copy and possibly overwrite immutable resources such as UUID.
+ public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, String path, boolean copyImmutableResource) throws IOException {
ResourceStore src = ResourceStore.getStore(srcConfig);
ResourceStore dst = ResourceStore.getStore(dstConfig);
logger.info("Copy from {} to {}", src, dst);
- copyR(src, dst, path);
+ copyR(src, dst, path, copyImmutableResource);
}
- public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths) throws IOException {
+ public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths) throws IOException {
+ copy(srcConfig, dstConfig, paths, false);
+ }
+
+ //Do NOT invoke this method directly, unless you want to copy and possibly overwrite immutable resources such as UUID.
+ public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths, boolean copyImmutableResource) throws IOException {
ResourceStore src = ResourceStore.getStore(srcConfig);
ResourceStore dst = ResourceStore.getStore(dstConfig);
logger.info("Copy from {} to {}", src, dst);
for (String path : paths) {
- copyR(src, dst, path);
+ copyR(src, dst, path, copyImmutableResource);
}
}
+
public static void copy(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
- copy(srcConfig, dstConfig, "/");
+ copy(srcConfig, dstConfig, false);
}
-
- public static void copyR(ResourceStore src, ResourceStore dst, String path) throws IOException {
+
+ //Do NOT invoke this method directly, unless you want to copy and possibly overwrite immutable resources such as UUID.
+ public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, boolean copyImmutableResource) throws IOException {
+ copy(srcConfig, dstConfig, "/", copyImmutableResource);
+ }
+
+ public static void copyR(ResourceStore src, ResourceStore dst, String path, boolean copyImmutableResource) throws IOException {
+
NavigableSet<String> children = src.listResources(path);
if (children == null) {
// case of resource (not a folder)
- if (matchFilter(path)) {
+ if (copyImmutableResource || matchFilter(path)) {
try {
RawResource res = src.getResource(path);
if (res != null) {
@@ -197,8 +214,9 @@ public class ResourceTool {
} else {
// case of folder
for (String child : children)
- copyR(src, dst, child);
+ copyR(src, dst, child, copyImmutableResource);
}
+
}
private static boolean matchFilter(String path) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/071f3b92/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 30bbb5e..bc8f34f 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -232,11 +232,11 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
KylinConfig srcConfig = KylinConfig.getInstanceFromEnv();
KylinConfig dstConfig = KylinConfig.createInstanceFromUri(dest);
- ResourceTool.copy(srcConfig, dstConfig, Lists.newArrayList(requiredResources));
+ ResourceTool.copy(srcConfig, dstConfig, Lists.newArrayList(requiredResources), true);
for (String r : optionalResources) {
try {
- ResourceTool.copy(srcConfig, dstConfig, Lists.newArrayList(r));
+ ResourceTool.copy(srcConfig, dstConfig, Lists.newArrayList(r), true);
} catch (Exception e) {
logger.warn("Exception when copying optional resource {}. May be caused by resource missing. skip it.", r);
}
[19/50] kylin git commit: minor,
disallow empty joint and hierarchy groups
Posted by li...@apache.org.
minor, disallow empty joint and hierarchy groups
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4d2a548a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4d2a548a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4d2a548a
Branch: refs/heads/master
Commit: 4d2a548accba5aa70ad7c2bcc405882a44fe9aae
Parents: 361b58d
Author: lidongsjtu <li...@apache.org>
Authored: Mon Jun 19 18:50:56 2017 +0800
Committer: Roger Shi <ro...@gmail.com>
Committed: Mon Jun 19 18:56:33 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/model/CubeDesc.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d2a548a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 4318570..91f985c 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -707,11 +707,11 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
throw new IllegalStateException("Aggregation group " + index + " hierarchy dimensions overlap with joint dimensions: " + ensureOrder(CollectionUtils.intersection(hierarchyDims, jointDims)));
}
- if (hasSingle(hierarchyDimsList)) {
+ if (hasSingleOrNone(hierarchyDimsList)) {
logger.error("Aggregation group " + index + " require at least 2 dimensions in a hierarchy");
throw new IllegalStateException("Aggregation group " + index + " require at least 2 dimensions in a hierarchy.");
}
- if (hasSingle(jointDimsList)) {
+ if (hasSingleOrNone(jointDimsList)) {
logger.error("Aggregation group " + index + " require at least 2 dimensions in a joint");
throw new IllegalStateException("Aggregation group " + index + " require at least 2 dimensions in a joint");
}
@@ -753,15 +753,15 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
}
}
- private boolean hasSingle(ArrayList<Set<String>> dimsList) {
- boolean hasSingle = false;
+ private boolean hasSingleOrNone(ArrayList<Set<String>> dimsList) {
+ boolean hasSingleOrNone = false;
for (Set<String> dims : dimsList) {
- if (dims.size() == 1) {
- hasSingle = true;
+ if (dims.size() <= 1) {
+ hasSingleOrNone = true;
break;
}
}
- return hasSingle;
+ return hasSingleOrNone;
}
private Pair<Boolean, Set<String>> hasOverlap(ArrayList<Set<String>> dimsList, Set<String> Dims) {
[28/50] kylin git commit: #1203 Move bin/setenv.sh to conf
Posted by li...@apache.org.
#1203 Move bin/setenv.sh to conf
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/57c1d5e0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/57c1d5e0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/57c1d5e0
Branch: refs/heads/master
Commit: 57c1d5e03f5cf07ccfb02f7711375c4aa72dd7a6
Parents: cfac81b
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jun 22 15:24:20 2017 +0800
Committer: Roger Shi <ro...@gmail.com>
Committed: Thu Jun 22 17:59:14 2017 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 5 +++++
build/bin/setenv.sh | 57 -----------------------------------------------
build/conf/setenv.sh | 57 +++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 62 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/57c1d5e0/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index a5808fd..08c9075 100755
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -41,8 +41,13 @@ function retrieveDependency() {
#retrive $KYLIN_EXTRA_START_OPTS
if [ -f "${dir}/setenv.sh" ]; then
+ echo "WARNING: ${dir}/setenv.sh is deprecated and ignored, please remove it and use ${KYLIN_HOME}/conf/setenv.sh instead"
source ${dir}/setenv.sh
fi
+
+ if [ -f "${KYLIN_HOME}/conf/setenv.sh" ]; then
+ source ${KYLIN_HOME}/conf/setenv.sh
+ fi
export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}:${kafka_dependency}:${spark_dependency}
http://git-wip-us.apache.org/repos/asf/kylin/blob/57c1d5e0/build/bin/setenv.sh
----------------------------------------------------------------------
diff --git a/build/bin/setenv.sh b/build/bin/setenv.sh
deleted file mode 100755
index 0e9b185..0000000
--- a/build/bin/setenv.sh
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# source me
-
-# (if your're deploying KYLIN on a powerful server and want to replace the default conservative settings)
-# uncomment following to for it to take effect
-export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -Xss1024K -XX:MaxPermSize=128M -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$KYLIN_HOME/logs/kylin.gc.$$ -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M"
-# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError"
-
-# uncomment following to for it to take effect(the values need adjusting to fit your env)
-# export KYLIN_DEBUG_SETTINGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
-
-# uncomment following to for it to take effect(the values need adjusting to fit your env)
-# export KYLIN_LD_LIBRARY_SETTINGS="-Djava.library.path=/apache/hadoop/lib/native/Linux-amd64-64"
-
-export KYLIN_EXTRA_START_OPTS=""
-
-if [ ! -z "${KYLIN_JVM_SETTINGS}" ]
-then
- verbose "KYLIN_JVM_SETTINGS is ${KYLIN_JVM_SETTINGS}"
- KYLIN_EXTRA_START_OPTS="${KYLIN_JVM_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
-else
- verbose "KYLIN_JVM_SETTINGS is not set, using default jvm settings: ${KYLIN_JVM_SETTINGS}"
-fi
-
-if [ ! -z "${KYLIN_DEBUG_SETTINGS}" ]
-then
- verbose "KYLIN_DEBUG_SETTINGS is ${KYLIN_DEBUG_SETTINGS}"
- KYLIN_EXTRA_START_OPTS="${KYLIN_DEBUG_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
-else
- verbose "KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging"
-fi
-
-if [ ! -z "${KYLIN_LD_LIBRARY_SETTINGS}" ]
-then
- verbose "KYLIN_LD_LIBRARY_SETTINGS is ${KYLIN_LD_LIBRARY_SETTINGS}"
- KYLIN_EXTRA_START_OPTS="${KYLIN_LD_LIBRARY_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
-else
- verbose "KYLIN_LD_LIBRARY_SETTINGS is not set, it is okay unless you want to specify your own native path"
-fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/57c1d5e0/build/conf/setenv.sh
----------------------------------------------------------------------
diff --git a/build/conf/setenv.sh b/build/conf/setenv.sh
new file mode 100755
index 0000000..0e9b185
--- /dev/null
+++ b/build/conf/setenv.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# source me
+
+# (if your're deploying KYLIN on a powerful server and want to replace the default conservative settings)
+# uncomment following to for it to take effect
+export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -Xss1024K -XX:MaxPermSize=128M -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$KYLIN_HOME/logs/kylin.gc.$$ -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M"
+# export KYLIN_JVM_SETTINGS="-Xms16g -Xmx16g -XX:MaxPermSize=512m -XX:NewSize=3g -XX:MaxNewSize=3g -XX:SurvivorRatio=4 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -XX:+HeapDumpOnOutOfMemoryError"
+
+# uncomment following to for it to take effect(the values need adjusting to fit your env)
+# export KYLIN_DEBUG_SETTINGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
+
+# uncomment following to for it to take effect(the values need adjusting to fit your env)
+# export KYLIN_LD_LIBRARY_SETTINGS="-Djava.library.path=/apache/hadoop/lib/native/Linux-amd64-64"
+
+export KYLIN_EXTRA_START_OPTS=""
+
+if [ ! -z "${KYLIN_JVM_SETTINGS}" ]
+then
+ verbose "KYLIN_JVM_SETTINGS is ${KYLIN_JVM_SETTINGS}"
+ KYLIN_EXTRA_START_OPTS="${KYLIN_JVM_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
+else
+ verbose "KYLIN_JVM_SETTINGS is not set, using default jvm settings: ${KYLIN_JVM_SETTINGS}"
+fi
+
+if [ ! -z "${KYLIN_DEBUG_SETTINGS}" ]
+then
+ verbose "KYLIN_DEBUG_SETTINGS is ${KYLIN_DEBUG_SETTINGS}"
+ KYLIN_EXTRA_START_OPTS="${KYLIN_DEBUG_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
+else
+ verbose "KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging"
+fi
+
+if [ ! -z "${KYLIN_LD_LIBRARY_SETTINGS}" ]
+then
+ verbose "KYLIN_LD_LIBRARY_SETTINGS is ${KYLIN_LD_LIBRARY_SETTINGS}"
+ KYLIN_EXTRA_START_OPTS="${KYLIN_LD_LIBRARY_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
+else
+ verbose "KYLIN_LD_LIBRARY_SETTINGS is not set, it is okay unless you want to specify your own native path"
+fi
[49/50] kylin git commit: KYLIN-2686 The same project's computed
column's definition can not be same
Posted by li...@apache.org.
KYLIN-2686 The same project's computed column's definition can not be same
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a0b7e74c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a0b7e74c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a0b7e74c
Branch: refs/heads/master
Commit: a0b7e74c16718bb7bab3eb093d6953c393f3444a
Parents: b2fc2c2
Author: Aron.tao <24...@qq.com>
Authored: Wed Jun 28 12:41:34 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Wed Jun 28 14:04:24 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/model/DataModelDesc.java | 43 ++++++++++++++------
1 file changed, 31 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a0b7e74c/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 7e08f1c..7de955e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -471,33 +471,52 @@ public class DataModelDesc extends RootPersistentEntity {
}
private void initComputedColumns(Map<String, CCInfo> ccInfoMap) {
+ if (ccInfoMap == null) {
+ logger.error("cc info map is null");
+ }
+
Set<String> ccSet = Sets.newHashSet();//make sure cc name does not duplicate within this model
- for (ComputedColumnDesc computedColumnDesc : this.computedColumnDescs) {
- computedColumnDesc.init();
+ for (ComputedColumnDesc thisCCDesc : this.computedColumnDescs) {
+ thisCCDesc.init();
+ String thisCCName = thisCCDesc.getFullName();
- if (ccSet.contains(computedColumnDesc.getFullName())) {
- throw new IllegalArgumentException(
- String.format("More than one computed column named %s exist in model %s",
- computedColumnDesc.getFullName(), this.getName()));
+ if (ccSet.contains(thisCCName)) {
+ throw new IllegalArgumentException(String.format(
+ "More than one computed column named %s exist in model %s", thisCCName, this.getName()));
} else {
- ccSet.add(computedColumnDesc.getFullName());
+ ccSet.add(thisCCName);
}
- CCInfo other = ccInfoMap.get(computedColumnDesc.getFullName());
+ CCInfo other = ccInfoMap.get(thisCCName);
if (other == null || (other.getDataModelDescs().size() == 1 && other.getDataModelDescs().contains(this))) {
- ccInfoMap.put(computedColumnDesc.getFullName(),
- new CCInfo(computedColumnDesc, Sets.<DataModelDesc> newHashSet(this)));
- } else if (other.getComputedColumnDesc().equals(computedColumnDesc)) {
+ //check whether two computer columns's definition is the same.
+ for (CCInfo sysCCInfo : ccInfoMap.values()) {
+ String definition0 = thisCCDesc.getExpression();
+ String definition1 = sysCCInfo.getComputedColumnDesc().getExpression();
+ if (isTwoCCDefinitionEquals(definition0, definition1)) {
+ throw new IllegalStateException(String.format(
+ "Computed column %s'definition: %s is already defined in other models: %s. Please change another definition, or try to keep consistent definition",
+ thisCCName, definition0, sysCCInfo.getDataModelDescs()));
+ }
+ }
+ ccInfoMap.put(thisCCName, new CCInfo(thisCCDesc, Sets.<DataModelDesc> newHashSet(this)));
+ } else if (other.getComputedColumnDesc().equals(thisCCDesc)) {
other.getDataModelDescs().add(this);
} else {
throw new IllegalStateException(String.format(
"Computed column named %s is already defined in other models: %s. Please change another name, or try to keep consistent definition", //
- computedColumnDesc.getFullName(), other.getDataModelDescs()));
+ thisCCName, other.getDataModelDescs()));
}
}
}
+ private boolean isTwoCCDefinitionEquals(String definition0, String definition1) {
+ definition0 = definition0.replaceAll("\\s*", "");
+ definition1 = definition1.replaceAll("\\s*", "");
+ return definition0.equalsIgnoreCase(definition1);
+ }
+
private void initJoinColumns() {
for (JoinTableDesc joinTable : joinTables) {
[09/50] kylin git commit: refine stats and cubing job name (#1215)
Posted by li...@apache.org.
refine stats and cubing job name (#1215)
* minor, refine stats job name
* minor, rename CubingJob name
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1c3329a8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1c3329a8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1c3329a8
Branch: refs/heads/master
Commit: 1c3329a89894b0825b07d58b7dcbafd983ebc036
Parents: d659bad
Author: 成 <ch...@kyligence.io>
Authored: Fri Jun 16 16:11:45 2017 +0800
Committer: luguosheng1314 <55...@qq.com>
Committed: Fri Jun 16 16:11:45 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/engine/mr/CubingJob.java | 27 +++++++++++++-------
1 file changed, 18 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1c3329a8/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
index 5aa7d72..466f706 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/CubingJob.java
@@ -80,11 +80,13 @@ public class CubingJob extends DefaultChainedExecutable {
private static CubingJob initCubingJob(CubeSegment seg, String jobType, String submitter, JobEngineConfig config) {
KylinConfig kylinConfig = config.getConfig();
CubeInstance cube = seg.getCubeInstance();
- List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(), cube.getName());
+ List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
+ cube.getName());
if (projList == null || projList.size() == 0) {
throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
} else if (projList.size() >= 2) {
- String msg = "Find more than one project containing the cube " + cube.getName() + ". It does't meet the uniqueness requirement!!! ";
+ String msg = "Find more than one project containing the cube " + cube.getName()
+ + ". It does't meet the uniqueness requirement!!! ";
if (!config.getConfig().allowCubeAppearInMultipleProjects()) {
throw new RuntimeException(msg);
} else {
@@ -99,7 +101,8 @@ public class CubingJob extends DefaultChainedExecutable {
result.setProjectName(projList.get(0).getName());
CubingExecutableUtil.setCubeName(seg.getCubeInstance().getName(), result.getParams());
CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
- result.setName(seg.getCubeInstance().getName() + " - " + seg.getName() + " - " + jobType + " - " + format.format(new Date(System.currentTimeMillis())));
+ result.setName(jobType + " CUBE - " + seg.getCubeInstance().getName() + " - " + seg.getName() + " - "
+ + format.format(new Date(System.currentTimeMillis())));
result.setSubmitter(submitter);
result.setNotifyList(seg.getCubeInstance().getDescriptor().getNotifyList());
return result;
@@ -127,11 +130,13 @@ public class CubingJob extends DefaultChainedExecutable {
@Override
protected Pair<String, String> formatNotifications(ExecutableContext context, ExecutableState state) {
- CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig()).getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig())
+ .getCube(CubingExecutableUtil.getCubeName(this.getParams()));
final Output output = getManager().getOutput(getId());
String logMsg;
state = output.getState();
- if (state != ExecutableState.ERROR && !cubeInstance.getDescriptor().getStatusNeedNotify().contains(state.toString())) {
+ if (state != ExecutableState.ERROR
+ && !cubeInstance.getDescriptor().getStatusNeedNotify().contains(state.toString())) {
logger.info("state:" + state + " no need to notify users");
return null;
}
@@ -160,7 +165,8 @@ public class CubingJob extends DefaultChainedExecutable {
content = content.replaceAll("\\$\\{mr_waiting\\}", getMapReduceWaitTime() / 60000 + "mins");
content = content.replaceAll("\\$\\{last_update_time\\}", new Date(getLastModified()).toString());
content = content.replaceAll("\\$\\{submitter\\}", StringUtil.noBlank(getSubmitter(), "missing submitter"));
- content = content.replaceAll("\\$\\{error_log\\}", Matcher.quoteReplacement(StringUtil.noBlank(logMsg, "no error message")));
+ content = content.replaceAll("\\$\\{error_log\\}",
+ Matcher.quoteReplacement(StringUtil.noBlank(logMsg, "no error message")));
try {
InetAddress inetAddress = InetAddress.getLocalHost();
@@ -169,7 +175,8 @@ public class CubingJob extends DefaultChainedExecutable {
logger.warn(e.getLocalizedMessage(), e);
}
- String title = "[" + state.toString() + "] - [" + getDeployEnvName() + "] - [" + getProjectName() + "] - " + CubingExecutableUtil.getCubeName(this.getParams());
+ String title = "[" + state.toString() + "] - [" + getDeployEnvName() + "] - [" + getProjectName() + "] - "
+ + CubingExecutableUtil.getCubeName(this.getParams());
return Pair.of(title, content);
}
@@ -196,7 +203,8 @@ public class CubingJob extends DefaultChainedExecutable {
*/
@Override
protected void handleMetaDataPersistException(Exception exception) {
- String title = "[ERROR] - [" + getDeployEnvName() + "] - [" + getProjectName() + "] - " + CubingExecutableUtil.getCubeName(this.getParams());
+ String title = "[ERROR] - [" + getDeployEnvName() + "] - [" + getProjectName() + "] - "
+ + CubingExecutableUtil.getCubeName(this.getParams());
String content = ExecutableConstants.NOTIFY_EMAIL_TEMPLATE;
final String UNKNOWN = "UNKNOWN";
String errMsg = null;
@@ -217,7 +225,8 @@ public class CubingJob extends DefaultChainedExecutable {
content = content.replaceAll("\\$\\{mr_waiting\\}", UNKNOWN);
content = content.replaceAll("\\$\\{last_update_time\\}", UNKNOWN);
content = content.replaceAll("\\$\\{submitter\\}", StringUtil.noBlank(getSubmitter(), "missing submitter"));
- content = content.replaceAll("\\$\\{error_log\\}", Matcher.quoteReplacement(StringUtil.noBlank(errMsg, "no error message")));
+ content = content.replaceAll("\\$\\{error_log\\}",
+ Matcher.quoteReplacement(StringUtil.noBlank(errMsg, "no error message")));
try {
InetAddress inetAddress = InetAddress.getLocalHost();
[23/50] kylin git commit: fix AccessService not returning parent obj
ACL
Posted by li...@apache.org.
fix AccessService not returning parent obj ACL
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73faf991
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73faf991
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73faf991
Branch: refs/heads/master
Commit: 73faf9917a788a7875691c019501f623e91c8107
Parents: 73bd033
Author: Li Yang <li...@apache.org>
Authored: Tue Jun 20 17:37:49 2017 +0800
Committer: Roger Shi <ro...@gmail.com>
Committed: Tue Jun 20 17:58:52 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/service/AccessService.java | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/73faf991/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 9ae372a..8b96635 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -19,7 +19,6 @@
package org.apache.kylin.rest.service;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import org.apache.kylin.common.persistence.AclEntity;
@@ -298,18 +297,16 @@ public class AccessService {
}
public List<AccessEntryResponse> generateAceResponses(Acl acl) {
- if (null == acl) {
- return Collections.emptyList();
- }
- List<AccessEntryResponse> accessControlEntities = new ArrayList<AccessEntryResponse>();
+ List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
- // Cause there is a circle reference in AccessControlEntry, it needs to
- // set acl to null as a workaround.
- for (AccessControlEntry ace : acl.getEntries()) {
- accessControlEntities.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
+ while (acl != null) {
+ for (AccessControlEntry ace : acl.getEntries()) {
+ result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
+ }
+ acl = acl.getParentAcl();
}
- return accessControlEntities;
+ return result;
}
/**
@@ -322,7 +319,8 @@ public class AccessService {
Message msg = MsgPicker.getMsg();
// Can't revoke admin permission from domain object owner
- if (acl.getOwner().equals(acl.getEntries().get(indexOfAce).getSid()) && BasePermission.ADMINISTRATION.equals(acl.getEntries().get(indexOfAce).getPermission())) {
+ if (acl.getOwner().equals(acl.getEntries().get(indexOfAce).getSid())
+ && BasePermission.ADMINISTRATION.equals(acl.getEntries().get(indexOfAce).getPermission())) {
throw new ForbiddenException(msg.getREVOKE_ADMIN_PERMISSION());
}
}
[30/50] kylin git commit: minor,
introduce ManagedUser to replace Spring User
Posted by li...@apache.org.
minor, introduce ManagedUser to replace Spring User
temp
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/cf2f27a3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/cf2f27a3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/cf2f27a3
Branch: refs/heads/master
Commit: cf2f27a3a8ffc41d5d909ff2b4fd38887305fe9d
Parents: 071f3b9
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Jun 22 14:52:28 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 22 18:26:32 2017 +0800
----------------------------------------------------------------------
.../rest/controller2/QueryControllerV2.java | 6 +
.../security/KylinAuthenticationProvider.java | 16 +-
.../kylin/rest/security/LdapProvider.java | 107 ---------
.../apache/kylin/rest/security/ManagedUser.java | 235 +++++++++++++++++++
.../rest/service/AclTableMigrationTool.java | 64 ++---
.../rest/service/UserGrantedAuthority.java | 5 +
.../org/apache/kylin/rest/service/UserInfo.java | 82 -------
.../apache/kylin/rest/service/UserService.java | 48 ++--
server/src/main/resources/kylinSecurity.xml | 4 +-
.../rest/controller/UserControllerTest.java | 4 +-
.../kylin/rest/service/ServiceTestBase.java | 15 +-
.../kylin/rest/service/UserServiceTest.java | 8 +-
12 files changed, 318 insertions(+), 276 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
index a641a53..a1b65a0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/QueryControllerV2.java
@@ -23,10 +23,13 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import javax.servlet.http.HttpServletResponse;
+import com.google.common.collect.Maps;
import org.apache.commons.io.IOUtils;
+import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.rest.controller.BasicController;
import org.apache.kylin.rest.exception.InternalErrorException;
@@ -84,6 +87,9 @@ public class QueryControllerV2 extends BasicController {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
public EnvelopeResponse prepareQueryV2(@RequestBody PrepareSqlRequest sqlRequest) {
+ Map<String, String> toggles = Maps.newHashMap();
+ toggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
+ BackdoorToggles.addToggles(toggles);
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, queryService.doQueryWithCache(sqlRequest), "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
index dc475c9..7322b84 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
@@ -18,8 +18,6 @@
package org.apache.kylin.rest.security;
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.rest.service.UserService;
import org.slf4j.Logger;
@@ -30,11 +28,13 @@ import org.springframework.security.authentication.AuthenticationProvider;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.AuthenticationException;
import org.springframework.security.core.context.SecurityContextHolder;
-import org.springframework.security.core.userdetails.User;
import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.util.Assert;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+
import net.sf.ehcache.Cache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
@@ -87,17 +87,19 @@ public class KylinAuthenticationProvider implements AuthenticationProvider {
logger.debug("Authenticated user " + authed.toString());
- UserDetails user;
+ ManagedUser user;
if (authed.getDetails() == null) {
//authed.setAuthenticated(false);
- throw new UsernameNotFoundException("User not found in LDAP, check whether he/she has been added to the groups.");
+ throw new UsernameNotFoundException(
+ "User not found in LDAP, check whether he/she has been added to the groups.");
}
if (authed.getDetails() instanceof UserDetails) {
- user = (UserDetails) authed.getDetails();
+ UserDetails details = (UserDetails) authed.getDetails();
+ user = new ManagedUser(details.getUsername(), details.getPassword(), false, details.getAuthorities());
} else {
- user = new User(authentication.getName(), "skippped-ldap", authed.getAuthorities());
+ user = new ManagedUser(authentication.getName(), "skippped-ldap", false, authed.getAuthorities());
}
Assert.notNull(user, "The UserDetail is null.");
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server-base/src/main/java/org/apache/kylin/rest/security/LdapProvider.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/LdapProvider.java b/server-base/src/main/java/org/apache/kylin/rest/security/LdapProvider.java
deleted file mode 100644
index c76301b..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/security/LdapProvider.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.security;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-
-import org.apache.kylin.rest.service.UserService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.security.core.Authentication;
-import org.springframework.security.core.AuthenticationException;
-import org.springframework.security.core.context.SecurityContextHolder;
-import org.springframework.security.core.userdetails.User;
-import org.springframework.security.core.userdetails.UserDetails;
-import org.springframework.security.ldap.authentication.LdapAuthenticationProvider;
-import org.springframework.security.ldap.authentication.LdapAuthenticator;
-import org.springframework.security.ldap.userdetails.LdapAuthoritiesPopulator;
-
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Element;
-
-/**
- * @author xduo
- * @deprecated replaced by KylinAuthenticationProvider
- *
- */
-public class LdapProvider extends LdapAuthenticationProvider {
-
- private static final Logger logger = LoggerFactory.getLogger(LdapProvider.class);
-
- @Autowired
- @Qualifier("userService")
- UserService userService;
-
- @Autowired
- private CacheManager cacheManager;
-
- MessageDigest md = null;
-
- /**
- * @param authenticator
- * @param authoritiesPopulator
- */
- public LdapProvider(LdapAuthenticator authenticator, LdapAuthoritiesPopulator authoritiesPopulator) {
- super(authenticator, authoritiesPopulator);
-
- try {
- md = MessageDigest.getInstance("MD5");
- } catch (NoSuchAlgorithmException e) {
- throw new RuntimeException("Failed to init Message Digest ", e);
- }
- }
-
- @Override
- public Authentication authenticate(Authentication authentication) throws AuthenticationException {
- Authentication authed = null;
- Cache userCache = cacheManager.getCache("UserCache");
- md.reset();
- byte[] hashKey = md.digest((authentication.getName() + authentication.getCredentials()).getBytes());
- String userKey = Arrays.toString(hashKey);
-
- Element authedUser = userCache.get(userKey);
- if (null != authedUser) {
- authed = (Authentication) authedUser.getObjectValue();
- SecurityContextHolder.getContext().setAuthentication(authed);
- } else {
- try {
- authed = super.authenticate(authentication);
- userCache.put(new Element(userKey, authed));
- } catch (AuthenticationException e) {
- logger.error("Failed to auth user: " + authentication.getName(), e);
- throw e;
- }
-
- UserDetails user = new User(authentication.getName(), "skippped-ldap", authed.getAuthorities());
-
- if (!userService.userExists(authentication.getName())) {
- userService.createUser(user);
- } else {
- userService.updateUser(user);
- }
- }
-
- return authed;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java b/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
new file mode 100644
index 0000000..4805d5c
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.security;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.rest.service.UserGrantedAuthority;
+import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.userdetails.UserDetails;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Lists;
+
+@SuppressWarnings("serial")
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
+public class ManagedUser extends RootPersistentEntity implements UserDetails {
+
+ @JsonProperty
+ private String username;
+ @JsonProperty
+ private String password;
+ @JsonProperty
+ private List<String> authorities = Lists.newArrayList();
+ @JsonProperty
+ private boolean disabled = false;
+ @JsonProperty
+ private boolean defaultPassword = false;
+ @JsonProperty
+ private boolean locked = false;
+ @JsonProperty
+ private long lockedTime = 0L;
+ @JsonProperty
+ private int wrongTime = 0;
+
+ private Boolean legacyCatered = false;
+ //DISABLED_ROLE is a ancient way to represent disabled user
+ //now we no longer support such way, however legacy metadata may still contain it
+ private static final String DISABLED_ROLE = "--disabled--";
+
+ //this is computed
+ private List<UserGrantedAuthority> grantedAuthorities = null;
+
+ public ManagedUser() {
+ }
+
+ public ManagedUser(String username, String password, Boolean defaultPassword, String... authorities) {
+ this.username = username;
+ this.password = password;
+ this.setDefaultPassword(defaultPassword);
+
+ this.authorities = Lists.newArrayList(authorities);
+ this.grantedAuthorities = null;
+ }
+
+ public ManagedUser(String username, String password, Boolean defaultPassword,
+ Collection<? extends GrantedAuthority> grantedAuthorities) {
+ this.username = username;
+ this.password = password;
+ this.setDefaultPassword(defaultPassword);
+
+ this.setGrantedAuthorities(grantedAuthorities);
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public void setUsername(String userName) {
+ this.username = userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ private void caterLegacy() {
+ if (!legacyCatered) {
+ synchronized (legacyCatered) {
+ Iterator<String> iterator = authorities.iterator();
+ while (iterator.hasNext()) {
+ if (DISABLED_ROLE.equals(iterator.next())) {
+ iterator.remove();
+ this.disabled = true;
+ }
+ }
+ legacyCatered = true;
+ }
+ }
+ }
+
+ public List<UserGrantedAuthority> getAuthorities() {
+ caterLegacy();
+ if (grantedAuthorities == null) {
+ grantedAuthorities = Lists.newArrayList();
+ for (String a : authorities) {
+ this.grantedAuthorities.add(new UserGrantedAuthority(a));
+ }
+ }
+ return grantedAuthorities;
+ }
+
+ public void setGrantedAuthorities(Collection<? extends GrantedAuthority> grantedAuthorities) {
+ this.authorities = Lists
+ .newArrayList(Collections2.transform(grantedAuthorities, new Function<GrantedAuthority, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable GrantedAuthority input) {
+ return input.getAuthority();
+ }
+ }));
+ this.grantedAuthorities = null;
+ }
+
+ public boolean isDisabled() {
+ caterLegacy();
+ return disabled;
+ }
+
+ public void setDisabled(boolean disabled) {
+ this.disabled = disabled;
+ }
+
+ public boolean isDefaultPassword() {
+ return defaultPassword;
+ }
+
+ public void setDefaultPassword(boolean defaultPassword) {
+ this.defaultPassword = defaultPassword;
+ }
+
+ public boolean isLocked() {
+ return locked;
+ }
+
+ public void setLocked(boolean locked) {
+ this.locked = locked;
+ }
+
+ public int getWrongTime() {
+ return wrongTime;
+ }
+
+ public long getLockedTime() {
+ return lockedTime;
+ }
+
+ public void increaseWrongTime() {
+ int wrongTime = this.getWrongTime();
+ if (wrongTime == 2) {
+ this.setLocked(true);
+ this.lockedTime = System.currentTimeMillis();
+ this.wrongTime = 0;
+ } else {
+ this.wrongTime = wrongTime + 1;
+ }
+ }
+
+ @Override
+ public boolean isAccountNonExpired() {
+ return true;
+ }
+
+ @Override
+ public boolean isAccountNonLocked() {
+ return !locked;
+ }
+
+ @Override
+ public boolean isCredentialsNonExpired() {
+ return true;
+ }
+
+ @Override
+ public boolean isEnabled() {
+ return !disabled;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((username == null) ? 0 : username.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ ManagedUser other = (ManagedUser) obj;
+ if (username == null) {
+ if (other.username != null)
+ return false;
+ } else if (!username.equals(other.username))
+ return false;
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "KapManagedUser [username=" + username + ", authorities=" + grantedAuthorities + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
index fc50410..64bac23 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclTableMigrationTool.java
@@ -19,7 +19,6 @@
package org.apache.kylin.rest.service;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -40,6 +39,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.StringEntity;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.rest.security.AclConstant;
+import org.apache.kylin.rest.security.ManagedUser;
import org.apache.kylin.rest.util.Serializer;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.HBaseResourceStore;
@@ -53,11 +53,13 @@ public class AclTableMigrationTool {
private static final Serializer<SidInfo> sidSerializer = new Serializer<SidInfo>(SidInfo.class);
- private static final Serializer<DomainObjectInfo> domainObjSerializer = new Serializer<DomainObjectInfo>(DomainObjectInfo.class);
+ private static final Serializer<DomainObjectInfo> domainObjSerializer = new Serializer<DomainObjectInfo>(
+ DomainObjectInfo.class);
private static final Serializer<AceInfo> aceSerializer = new Serializer<AceInfo>(AceInfo.class);
- private static final Serializer<UserGrantedAuthority[]> ugaSerializer = new Serializer<UserGrantedAuthority[]>(UserGrantedAuthority[].class);
+ private static final Serializer<UserGrantedAuthority[]> ugaSerializer = new Serializer<UserGrantedAuthority[]>(
+ UserGrantedAuthority[].class);
public static final String MIGRATE_OK_PREFIX = "MIGRATE_OK_";
@@ -69,7 +71,8 @@ public class AclTableMigrationTool {
return;
} else {
if (!kylinConfig.getServerMode().equals("all")) {
- throw new IllegalStateException("Please make sure that you have config kylin.server.mode=all before migrating data");
+ throw new IllegalStateException(
+ "Please make sure that you have config kylin.server.mode=all before migrating data");
}
logger.info("Start to migrate acl table data");
ResourceStore store = ResourceStore.getStore(kylinConfig);
@@ -144,11 +147,9 @@ public class AclTableMigrationTool {
return;
Result result = rs.next();
while (result != null) {
- User user = hbaseRowToUser(result);
- UserInfo userInfo = convert(user);
- store.deleteResource(UserService.getId(userInfo.getUsername()));
- store.putResource(UserService.getId(userInfo.getUsername()), userInfo, 0,
- UserService.SERIALIZER);
+ ManagedUser user = hbaseRowToUser(result);
+ store.deleteResource(UserService.getId(user.getUsername()));
+ store.putResource(UserService.getId(user.getUsername()), user, 0, UserService.SERIALIZER);
result = rs.next();
}
}
@@ -172,7 +173,8 @@ public class AclTableMigrationTool {
return store.exists(MIGRATE_OK_PREFIX + tableName);
}
- private void convertToResourceStore(KylinConfig kylinConfig, String tableName, ResourceStore store, ResultConverter converter) throws IOException {
+ private void convertToResourceStore(KylinConfig kylinConfig, String tableName, ResourceStore store,
+ ResultConverter converter) throws IOException {
Table table = null;
ResultScanner rs = null;
@@ -181,7 +183,8 @@ public class AclTableMigrationTool {
table = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(tableName));
rs = table.getScanner(scan);
converter.convertResult(rs, store);
- store.putResource(MIGRATE_OK_PREFIX + tableName, new StringEntity(tableName + " migrated"), StringEntity.serializer);
+ store.putResource(MIGRATE_OK_PREFIX + tableName, new StringEntity(tableName + " migrated"),
+ StringEntity.serializer);
} finally {
IOUtils.closeQuietly(rs);
IOUtils.closeQuietly(table);
@@ -190,7 +193,8 @@ public class AclTableMigrationTool {
}
private DomainObjectInfo getDomainObjectInfoFromRs(Result result) {
- String type = String.valueOf(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY), Bytes.toBytes(AclConstant.ACL_INFO_FAMILY_TYPE_COLUMN)));
+ String type = String.valueOf(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY),
+ Bytes.toBytes(AclConstant.ACL_INFO_FAMILY_TYPE_COLUMN)));
String id = new String(result.getRow());
DomainObjectInfo newInfo = new DomainObjectInfo();
newInfo.setId(id);
@@ -199,17 +203,20 @@ public class AclTableMigrationTool {
}
private DomainObjectInfo getParentDomainObjectInfoFromRs(Result result) throws IOException {
- DomainObjectInfo parentInfo = domainObjSerializer.deserialize(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY), Bytes.toBytes(AclConstant.ACL_INFO_FAMILY_PARENT_COLUMN)));
+ DomainObjectInfo parentInfo = domainObjSerializer.deserialize(result.getValue(
+ Bytes.toBytes(AclConstant.ACL_INFO_FAMILY), Bytes.toBytes(AclConstant.ACL_INFO_FAMILY_PARENT_COLUMN)));
return parentInfo;
}
private boolean getInheriting(Result result) {
- boolean entriesInheriting = Bytes.toBoolean(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY), Bytes.toBytes(AclConstant.ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN)));
+ boolean entriesInheriting = Bytes.toBoolean(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY),
+ Bytes.toBytes(AclConstant.ACL_INFO_FAMILY_ENTRY_INHERIT_COLUMN)));
return entriesInheriting;
}
private SidInfo getOwnerSidInfo(Result result) throws IOException {
- SidInfo owner = sidSerializer.deserialize(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY), Bytes.toBytes(AclConstant.ACL_INFO_FAMILY_OWNER_COLUMN)));
+ SidInfo owner = sidSerializer.deserialize(result.getValue(Bytes.toBytes(AclConstant.ACL_INFO_FAMILY),
+ Bytes.toBytes(AclConstant.ACL_INFO_FAMILY_OWNER_COLUMN)));
return owner;
}
@@ -226,27 +233,14 @@ public class AclTableMigrationTool {
return allAceInfoMap;
}
- private UserInfo convert(User user) {
- if (user == null)
- return null;
- UserInfo newInfo = new UserInfo();
- newInfo.setUsername(user.getUserName());
- newInfo.setPassword(user.getPassword());
- List<String> authorities = new ArrayList<>();
- for (String auth : user.getAuthorities()) {
- authorities.add(auth);
- }
- newInfo.setAuthorities(authorities);
- return newInfo;
- }
-
- private User hbaseRowToUser(Result result) throws JsonParseException, JsonMappingException, IOException {
+ private ManagedUser hbaseRowToUser(Result result) throws JsonParseException, JsonMappingException, IOException {
if (null == result || result.isEmpty())
return null;
String username = Bytes.toString(result.getRow());
- byte[] valueBytes = result.getValue(Bytes.toBytes(AclConstant.USER_AUTHORITY_FAMILY), Bytes.toBytes(AclConstant.USER_AUTHORITY_COLUMN));
+ byte[] valueBytes = result.getValue(Bytes.toBytes(AclConstant.USER_AUTHORITY_FAMILY),
+ Bytes.toBytes(AclConstant.USER_AUTHORITY_COLUMN));
UserGrantedAuthority[] deserialized = ugaSerializer.deserialize(valueBytes);
String password = "";
@@ -261,13 +255,7 @@ public class AclTableMigrationTool {
authorities = Arrays.asList(deserialized);
}
}
- List<String> authoritiesStr = new ArrayList<>();
- for (UserGrantedAuthority auth : authorities) {
- if (auth != null) {
- authoritiesStr.add(auth.getAuthority());
- }
- }
- return new User(username, password, authoritiesStr);
+ return new ManagedUser(username, password, false, authorities);
}
interface ResultConverter {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server-base/src/main/java/org/apache/kylin/rest/service/UserGrantedAuthority.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserGrantedAuthority.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserGrantedAuthority.java
index 4c2a392..1227177 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserGrantedAuthority.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserGrantedAuthority.java
@@ -20,10 +20,15 @@ package org.apache.kylin.rest.service;
import org.springframework.security.core.GrantedAuthority;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+@SuppressWarnings("serial")
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE, getterVisibility = JsonAutoDetect.Visibility.NONE, isGetterVisibility = JsonAutoDetect.Visibility.NONE, setterVisibility = JsonAutoDetect.Visibility.NONE)
public class UserGrantedAuthority implements GrantedAuthority {
private static final long serialVersionUID = -5128905636841891058L;
+ @JsonProperty
private String authority;
public UserGrantedAuthority() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server-base/src/main/java/org/apache/kylin/rest/service/UserInfo.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserInfo.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserInfo.java
deleted file mode 100644
index 644883d..0000000
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserInfo.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.rest.service;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.springframework.security.core.GrantedAuthority;
-import org.springframework.security.core.userdetails.UserDetails;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-
-@SuppressWarnings("serial")
-public class UserInfo extends RootPersistentEntity {
-
- @JsonProperty()
- private String username;
- @JsonProperty()
- private String password;
- @JsonProperty()
- private List<String> authorities = new ArrayList<>();
-
- public UserInfo(String username, String password, List<String> authorities) {
- this.username = username;
- this.password = password;
- this.authorities = authorities;
- }
-
- public UserInfo(UserDetails user) {
- this.username = user.getUsername();
- this.password = user.getPassword();
- for (GrantedAuthority a : user.getAuthorities()) {
- this.authorities.add(a.getAuthority());
- }
- }
-
- public UserInfo() {
- }
-
- public String getUsername() {
- return username;
- }
-
- public void setUsername(String username) {
- this.username = username;
- }
-
- public String getPassword() {
- return password;
- }
-
- public void setPassword(String password) {
- this.password = password;
- }
-
- public List<String> getAuthorities() {
- return authorities;
- }
-
- public void setAuthorities(List<String> authorities) {
- this.authorities = authorities;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index e803040..504c035 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -31,15 +31,17 @@ import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
+import org.apache.kylin.rest.security.ManagedUser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.core.GrantedAuthority;
-import org.springframework.security.core.userdetails.User;
import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.security.provisioning.UserDetailsManager;
import org.springframework.stereotype.Component;
+import com.google.common.base.Preconditions;
+
/**
*/
@Component("userService")
@@ -49,7 +51,7 @@ public class UserService implements UserDetailsManager {
public static final String DIR_PREFIX = "/user/";
- public static final Serializer<UserInfo> SERIALIZER = new JsonSerializer<>(UserInfo.class);
+ public static final Serializer<ManagedUser> SERIALIZER = new JsonSerializer<>(ManagedUser.class);
protected ResourceStore aclStore;
@@ -67,11 +69,13 @@ public class UserService implements UserDetailsManager {
@Override
//@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) --- DON'T DO THIS, CAUSES CIRCULAR DEPENDENCY BETWEEN UserService & AclService
public void updateUser(UserDetails user) {
+ Preconditions.checkState(user instanceof ManagedUser, "User {} is not ManagedUser", user);
+ ManagedUser managedUser = (ManagedUser) user;
try {
deleteUser(user.getUsername());
String id = getId(user.getUsername());
- aclStore.putResource(id, new UserInfo(user), 0, SERIALIZER);
- logger.debug("update user : {}", user.getUsername());
+ aclStore.putResource(id, managedUser, 0, SERIALIZER);
+ logger.trace("update user : {}", user.getUsername());
} catch (IOException e) {
throw new InternalErrorException(e);
}
@@ -82,7 +86,7 @@ public class UserService implements UserDetailsManager {
try {
String id = getId(userName);
aclStore.deleteResource(id);
- logger.debug("delete user : {}", userName);
+ logger.trace("delete user : {}", userName);
} catch (IOException e) {
throw new InternalErrorException(e);
}
@@ -96,23 +100,27 @@ public class UserService implements UserDetailsManager {
@Override
public boolean userExists(String userName) {
try {
- logger.debug("judge user exist: {}", userName);
+ logger.trace("judge user exist: {}", userName);
return aclStore.exists(getId(userName));
} catch (IOException e) {
throw new InternalErrorException(e);
}
}
+ /**
+ *
+ * @return a ManagedUser
+ */
@Override
public UserDetails loadUserByUsername(String userName) throws UsernameNotFoundException {
Message msg = MsgPicker.getMsg();
try {
- UserInfo userInfo = aclStore.getResource(getId(userName), UserInfo.class, SERIALIZER);
- if (userInfo == null) {
+ ManagedUser managedUser = aclStore.getResource(getId(userName), ManagedUser.class, SERIALIZER);
+ if (managedUser == null) {
throw new UsernameNotFoundException(String.format(msg.getUSER_NOT_FOUND(), userName));
}
- logger.debug("load user : {}", userName);
- return wrap(userInfo);
+ logger.trace("load user : {}", userName);
+ return managedUser;
} catch (IOException e) {
throw new InternalErrorException(e);
}
@@ -130,28 +138,12 @@ public class UserService implements UserDetailsManager {
return all;
}
- public List<UserDetails> listUsers() throws IOException {
- List<UserDetails> all = new ArrayList<UserDetails>();
- List<UserInfo> userInfos = aclStore.getAllResources(DIR_PREFIX, UserInfo.class, SERIALIZER);
- for (UserInfo info : userInfos) {
- all.add(wrap(info));
- }
- return all;
+ public List<ManagedUser> listUsers() throws IOException {
+ return aclStore.getAllResources(DIR_PREFIX, ManagedUser.class, SERIALIZER);
}
public static String getId(String userName) {
return DIR_PREFIX + userName;
}
- protected User wrap(UserInfo userInfo) {
- if (userInfo == null)
- return null;
- List<GrantedAuthority> authorities = new ArrayList<>();
- List<String> auths = userInfo.getAuthorities();
- for (String str : auths) {
- authorities.add(new UserGrantedAuthority(str));
- }
- return new User(userInfo.getUsername(), userInfo.getPassword(), authorities);
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server/src/main/resources/kylinSecurity.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/kylinSecurity.xml b/server/src/main/resources/kylinSecurity.xml
index 039bded..53ed511 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -246,7 +246,7 @@
<scr:intercept-url pattern="/api/admin*/**" access="hasRole('ROLE_ADMIN')"/>
<scr:intercept-url pattern="/api/**" access="isAuthenticated()"/>
- <scr:logout invalidate-session="true" delete-cookies="JSESSIONID"/>
+ <scr:logout invalidate-session="true" delete-cookies="JSESSIONID" logout-url="/j_spring_security_logout" logout-success-url="/." />
<scr:session-management session-fixation-protection="newSession"/>
</scr:http>
</beans>
@@ -288,7 +288,7 @@
<scr:intercept-url pattern="/api/admin*/**" access="hasRole('ROLE_ADMIN')"/>
<scr:intercept-url pattern="/api/**" access="isAuthenticated()"/>
- <scr:logout invalidate-session="true" delete-cookies="JSESSIONID"/>
+ <scr:logout invalidate-session="true" delete-cookies="JSESSIONID" logout-url="/j_spring_security_logout" logout-success-url="/." />
<scr:session-management session-fixation-protection="newSession"/>
</scr:http>
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
index 767aaf1..f6b4ae1 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/UserControllerTest.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.security.ManagedUser;
import org.apache.kylin.rest.service.ServiceTestBase;
import org.junit.Assert;
import org.junit.Before;
@@ -32,7 +33,6 @@ import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.context.SecurityContextHolder;
-import org.springframework.security.core.userdetails.User;
import org.springframework.security.core.userdetails.UserDetails;
/**
@@ -46,7 +46,7 @@ public class UserControllerTest extends ServiceTestBase {
public static void setupResource() {
staticCreateTestMetadata();
List<GrantedAuthority> authorities = new ArrayList<GrantedAuthority>();
- User user = new User("ADMIN", "ADMIN", authorities);
+ ManagedUser user = new ManagedUser("ADMIN", "ADMIN", false, authorities);
Authentication authentication = new TestingAuthenticationToken(user, "ADMIN", Constant.ROLE_ADMIN);
SecurityContextHolder.getContext().setAuthentication(authentication);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
index b45b27b..1d60a53 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java
@@ -24,6 +24,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.security.ManagedUser;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -35,7 +36,6 @@ import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.security.authentication.TestingAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
-import org.springframework.security.core.userdetails.User;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@@ -71,18 +71,19 @@ public class ServiceTestBase extends LocalFileMetadataTestCase {
Broadcaster.getInstance(config).notifyClearAll();
if (!userService.userExists("ADMIN")) {
- userService.createUser(new User("ADMIN", "KYLIN", Arrays.asList(//
- new UserGrantedAuthority(Constant.ROLE_ADMIN), new UserGrantedAuthority(Constant.ROLE_ANALYST), new UserGrantedAuthority(Constant.ROLE_MODELER))));
+ userService.createUser(new ManagedUser("ADMIN", "KYLIN", false, Arrays.asList(//
+ new UserGrantedAuthority(Constant.ROLE_ADMIN), new UserGrantedAuthority(Constant.ROLE_ANALYST),
+ new UserGrantedAuthority(Constant.ROLE_MODELER))));
}
if (!userService.userExists("MODELER")) {
- userService.createUser(new User("MODELER", "MODELER", Arrays.asList(//
- new UserGrantedAuthority(Constant.ROLE_ANALYST), new UserGrantedAuthority(Constant.ROLE_MODELER))));
+ userService.createUser(new ManagedUser("MODELER", "MODELER", false, Arrays.asList(//
+ new UserGrantedAuthority(Constant.ROLE_ANALYST), new UserGrantedAuthority(Constant.ROLE_MODELER))));
}
if (!userService.userExists("ANALYST")) {
- userService.createUser(new User("ANALYST", "ANALYST", Arrays.asList(//
- new UserGrantedAuthority(Constant.ROLE_ANALYST))));
+ userService.createUser(new ManagedUser("ANALYST", "ANALYST", false, Arrays.asList(//
+ new UserGrantedAuthority(Constant.ROLE_ANALYST))));
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/cf2f27a3/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
index 0d4b580..c49b552 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/UserServiceTest.java
@@ -23,13 +23,13 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.security.ManagedUser;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.authority.SimpleGrantedAuthority;
-import org.springframework.security.core.userdetails.User;
import org.springframework.security.core.userdetails.UserDetails;
/**
@@ -44,11 +44,12 @@ public class UserServiceTest extends ServiceTestBase {
@Test
public void testBasics() throws IOException {
userService.deleteUser("ADMIN");
+
Assert.assertTrue(!userService.userExists("ADMIN"));
List<GrantedAuthority> authorities = new ArrayList<GrantedAuthority>();
authorities.add(new SimpleGrantedAuthority(Constant.ROLE_ADMIN));
- User user = new User("ADMIN", "PWD", authorities);
+ ManagedUser user = new ManagedUser("ADMIN", "PWD", false, authorities);
userService.createUser(user);
Assert.assertTrue(userService.userExists("ADMIN"));
@@ -59,7 +60,8 @@ public class UserServiceTest extends ServiceTestBase {
Assert.assertEquals(Constant.ROLE_ADMIN, ud.getAuthorities().iterator().next().getAuthority());
Assert.assertEquals(1, ud.getAuthorities().size());
- Assert.assertTrue(userService.listUserAuthorities().contains(Constant.ROLE_ADMIN));
+ List<String> strings = userService.listUserAuthorities();
+ Assert.assertTrue(strings.contains(Constant.ROLE_ADMIN));
}
}
[22/50] kylin git commit: KYLIN-2662 report error when writing super
large dict cell
Posted by li...@apache.org.
KYLIN-2662 report error when writing super large dict cell
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73bd0339
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73bd0339
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73bd0339
Branch: refs/heads/master
Commit: 73bd0339fea74944e465921ed4283215b01e2c67
Parents: 247f3cd
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Jun 20 16:32:01 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Tue Jun 20 17:16:03 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/util/BytesUtil.java | 4 ++++
.../src/main/java/org/apache/kylin/dict/TrieDictionary.java | 5 ++++-
.../java/org/apache/kylin/dict/TrieDictionaryBuilder.java | 8 ++++++++
3 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/73bd0339/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index 49f1ee0..8cc559e 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -177,6 +177,10 @@ public class BytesUtil {
return r;
}
+ public static boolean isPositiveShort(int i) {
+ return (i & 0xFFFF7000) == 0;
+ }
+
// from WritableUtils
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/73bd0339/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 8849015..53daeb7 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -29,7 +29,6 @@ import java.io.ObjectOutputStream;
import java.io.PrintStream;
import java.util.Arrays;
-
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
@@ -101,6 +100,10 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
this.sizeNoValuesBeneath = headIn.read();
this.baseId = headIn.readShort();
this.maxValueLength = headIn.readShort();
+ if (maxValueLength < 0) {
+ throw new IllegalStateException("maxValueLength is negative (" + maxValueLength
+ + "). Dict value is too long, whose length is larger than " + Short.MAX_VALUE);
+ }
String converterName = headIn.readUTF();
if (converterName.isEmpty() == false)
http://git-wip-us.apache.org/repos/asf/kylin/blob/73bd0339/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
index 1750ac1..18169ca 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
@@ -442,7 +442,9 @@ public class TrieDictionaryBuilder<T> {
headOut.writeInt((int) stats.mbpn_footprint); // body size
headOut.write(sizeChildOffset);
headOut.write(sizeNoValuesBeneath);
+ positiveShortPreCheck(baseId, "baseId");
headOut.writeShort(baseId);
+ positiveShortPreCheck(stats.maxValueLength, "stats.maxValueLength");
headOut.writeShort(stats.maxValueLength);
headOut.writeUTF(bytesConverter == null ? "" : bytesConverter.getClass().getName());
headOut.close();
@@ -483,6 +485,12 @@ public class TrieDictionaryBuilder<T> {
return trieBytes;
}
+ private void positiveShortPreCheck(int i, String fieldName) {
+ if (!BytesUtil.isPositiveShort(i)) {
+ throw new IllegalStateException(fieldName + " is not positive short, usually caused by too long dict value.");
+ }
+ }
+
private void build_overwriteChildOffset(int parentOffset, int childOffset, int sizeChildOffset, byte[] trieBytes) {
int flags = (int) trieBytes[parentOffset] & (TrieDictionary.BIT_IS_LAST_CHILD | TrieDictionary.BIT_IS_END_OF_VALUE);
BytesUtil.writeUnsigned(childOffset, trieBytes, parentOffset, sizeChildOffset);
[18/50] kylin git commit: fix minor issues
Posted by li...@apache.org.
fix minor issues
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/361b58d6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/361b58d6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/361b58d6
Branch: refs/heads/master
Commit: 361b58d6dc5fc1add839952ecf1c687662577bc8
Parents: d67bf45
Author: Hongbin Ma <ma...@apache.org>
Authored: Mon Jun 19 15:34:16 2017 +0800
Committer: Ni Chunen <zj...@sjtu.org>
Committed: Mon Jun 19 16:39:05 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/jdbc/ITJDBCDriverTest.java | 16 ++++++++--------
.../kylin/rest/controller/QueryController.java | 16 ++++++++++++----
.../org/apache/kylin/rest/service/QueryService.java | 2 +-
3 files changed, 21 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/361b58d6/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index bdf61f8..ba293ae 100644
--- a/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -222,17 +222,15 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
public void testPreparedStatement() throws Exception {
Connection conn = getConnection();
- PreparedStatement statement = conn.prepareStatement("select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact " + "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME");
+ PreparedStatement statement = conn.prepareStatement(
+ "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact "
+ + "where LSTG_FORMAT_NAME = ? group by LSTG_FORMAT_NAME");
statement.setString(1, "FP-GTC");
ResultSet rs = statement.executeQuery();
- Assert.assertTrue(rs.next());
-
- String format_name = rs.getString(1);
-
- Assert.assertTrue("FP-GTC".equals(format_name));
+ Assert.assertFalse(rs.next());
rs.close();
statement.close();
@@ -242,7 +240,8 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
@Test
public void testResultSet() throws Exception {
- String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact \n" + " group by LSTG_FORMAT_NAME ";
+ String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact \n"
+ + " group by LSTG_FORMAT_NAME ";
Connection conn = getConnection();
Statement statement = conn.createStatement();
@@ -270,7 +269,8 @@ public class ITJDBCDriverTest extends HBaseMetadataTestCase {
@Test
public void testResultSetWithMaxRows() throws Exception {
- String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact \n" + " group by LSTG_FORMAT_NAME ";
+ String sql = "select LSTG_FORMAT_NAME, sum(price) as GMV, count(1) as TRANS_CNT from test_kylin_fact \n"
+ + " group by LSTG_FORMAT_NAME ";
Connection conn = getConnection();
Statement statement = conn.createStatement();
http://git-wip-us.apache.org/repos/asf/kylin/blob/361b58d6/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 35dcee4..61250ea 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -20,13 +20,14 @@ package org.apache.kylin.rest.controller;
import java.io.IOException;
import java.sql.SQLException;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Date;
import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpServletResponse;
-import com.google.common.collect.Maps;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
@@ -54,6 +55,8 @@ import org.supercsv.io.CsvListWriter;
import org.supercsv.io.ICsvListWriter;
import org.supercsv.prefs.CsvPreference;
+import com.google.common.collect.Maps;
+
/**
* Handle query requests.
*
@@ -82,7 +85,7 @@ public class QueryController extends BasicController {
Map<String, String> toggles = Maps.newHashMap();
toggles.put(BackdoorToggles.DEBUG_TOGGLE_PREPARE_ONLY, "true");
BackdoorToggles.addToggles(toggles);
-
+
return queryService.doQueryWithCache(sqlRequest);
}
@@ -90,7 +93,8 @@ public class QueryController extends BasicController {
@ResponseBody
public void saveQuery(@RequestBody SaveSqlRequest sqlRequest) throws IOException {
String creator = SecurityContextHolder.getContext().getAuthentication().getName();
- Query newQuery = new Query(sqlRequest.getName(), sqlRequest.getProject(), sqlRequest.getSql(), sqlRequest.getDescription());
+ Query newQuery = new Query(sqlRequest.getName(), sqlRequest.getProject(), sqlRequest.getSql(),
+ sqlRequest.getDescription());
queryService.saveQuery(creator, newQuery);
}
@@ -114,7 +118,11 @@ public class QueryController extends BasicController {
public void downloadQueryResult(@PathVariable String format, SQLRequest sqlRequest, HttpServletResponse response) {
SQLResponse result = queryService.doQueryWithCache(sqlRequest);
response.setContentType("text/" + format + ";charset=utf-8");
- response.setHeader("Content-Disposition", "attachment; filename=\"result." + format + "\"");
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmssSSS");
+ Date now = new Date();
+ String nowStr = sdf.format(now);
+ response.setHeader("Content-Disposition", "attachment; filename=\"" + nowStr + ".result." + format + "\"");
ICsvListWriter csvWriter = null;
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/361b58d6/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index bf32140..d6554fc 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -772,7 +772,7 @@ public class QueryService extends BasicService {
String columnName = field.getKey();
BasicSqlType basicSqlType = (BasicSqlType) field.getValue();
- columnMetas.add(new SelectedColumnMeta(false, config.caseSensitive(), false, false, basicSqlType.isNullable() ? 1 : 0, true, basicSqlType.getPrecision(), columnName, columnName, null, null, null, basicSqlType.getPrecision(), basicSqlType.getScale(), basicSqlType.getSqlTypeName().getJdbcOrdinal(), basicSqlType.getSqlTypeName().getName(), true, false, false));
+ columnMetas.add(new SelectedColumnMeta(false, config.caseSensitive(), false, false, basicSqlType.isNullable() ? 1 : 0, true, basicSqlType.getPrecision(), columnName, columnName, null, null, null, basicSqlType.getPrecision(), basicSqlType.getScale() < 0 ? 0 : basicSqlType.getScale(), basicSqlType.getSqlTypeName().getJdbcOrdinal(), basicSqlType.getSqlTypeName().getName(), true, false, false));
}
} else {
[47/50] kylin git commit: #488 (#1406)
Posted by li...@apache.org.
#488 (#1406)
* #488
* #488 some code refine
* #488
* #488
* #488
* #488
* #488
* #488
* #488
* #488
* #488
* code refine
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1acd066c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1acd066c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1acd066c
Branch: refs/heads/master
Commit: 1acd066cc0e5dc40e9547f25be7fcc992a1dd941
Parents: 05631b5
Author: luguosheng1314 <55...@qq.com>
Authored: Tue Jun 27 19:03:20 2017 +0800
Committer: GitHub <no...@github.com>
Committed: Tue Jun 27 19:03:20 2017 +0800
----------------------------------------------------------------------
webapp/app/partials/tables/loadStreamingTable.html | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1acd066c/webapp/app/partials/tables/loadStreamingTable.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/tables/loadStreamingTable.html b/webapp/app/partials/tables/loadStreamingTable.html
index 2f2a1ec..beac941 100644
--- a/webapp/app/partials/tables/loadStreamingTable.html
+++ b/webapp/app/partials/tables/loadStreamingTable.html
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
-->
-
<div class="modal-body streaming-source" style="height: 360px;">
<div class="col-xs-5">
<p class="text-info">
@@ -31,7 +30,6 @@
Source json invalid, Please correct your schema and generate again.
</small>
</div>
-
<div style="margin-bottom: 20px;">
<span class="label label-info">JSON</span>
</div>
@@ -41,7 +39,6 @@
mode:'json',
onLoad: streamingOnLoad
}">
-
</div>
</div>
<div class="col-xs-1" style="margin-top:300px;text-align:center;">
[44/50] kylin git commit: minor,
project renaming also copy ACL information
Posted by li...@apache.org.
minor, project renaming also copy ACL information
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/450845c6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/450845c6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/450845c6
Branch: refs/heads/master
Commit: 450845c6377c131db801f9eca9c4e062fd67fc0e
Parents: 6f8a91c
Author: Roger Shi <ro...@hotmail.com>
Authored: Mon Jun 26 21:19:43 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Tue Jun 27 11:18:03 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/project/ProjectManager.java | 52 +++++++++++---------
.../rest/controller/ProjectController.java | 6 ++-
.../rest/controller2/ProjectControllerV2.java | 10 ++--
.../java/org/apache/kylin/rest/msg/Message.java | 4 --
.../kylin/rest/service/ProjectService.java | 24 ++++++++-
5 files changed, 62 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/450845c6/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index d4b3ff4..213b136 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -47,6 +47,7 @@ import org.apache.kylin.metadata.realization.RealizationType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -216,35 +217,40 @@ public class ProjectManager {
}
}
- //update project itself
- public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc,
+ // rename project
+ public ProjectInstance renameProject(ProjectInstance project, String newName, String newDesc,
LinkedHashMap<String, String> overrideProps) throws IOException {
- if (!project.getName().equals(newName)) {
- ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc, overrideProps);
-
- newProject.setCreateTimeUTC(project.getCreateTimeUTC());
- newProject.recordUpdateTime(System.currentTimeMillis());
- newProject.setRealizationEntries(project.getRealizationEntries());
- newProject.setTables(project.getTables());
- newProject.setModels(project.getModels());
- newProject.setExtFilters(project.getExtFilters());
+ Preconditions.checkArgument(!project.getName().equals(newName));
+ ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc, overrideProps);
+
+ newProject.setUuid(project.getUuid());
+ newProject.setCreateTimeUTC(project.getCreateTimeUTC());
+ newProject.recordUpdateTime(System.currentTimeMillis());
+ newProject.setRealizationEntries(project.getRealizationEntries());
+ newProject.setTables(project.getTables());
+ newProject.setModels(project.getModels());
+ newProject.setExtFilters(project.getExtFilters());
+
+ removeProject(project);
+ updateProject(newProject);
- removeProject(project);
- updateProject(newProject);
+ return newProject;
+ }
- return newProject;
- } else {
- project.setName(newName);
- project.setDescription(newDesc);
- project.setOverrideKylinProps(overrideProps);
+ //update project itself
+ public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc,
+ LinkedHashMap<String, String> overrideProps) throws IOException {
+ Preconditions.checkArgument(project.getName().equals(newName));
+ project.setName(newName);
+ project.setDescription(newDesc);
+ project.setOverrideKylinProps(overrideProps);
- if (project.getUuid() == null)
- project.updateRandomUuid();
+ if (project.getUuid() == null)
+ project.updateRandomUuid();
- updateProject(project);
+ updateProject(project);
- return project;
- }
+ return project;
}
private void updateProject(ProjectInstance prj) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/450845c6/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
index 89c8b23..74e806e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
@@ -172,7 +172,11 @@ public class ProjectController extends BasicController {
throw new InternalErrorException("The project named " + formerProjectName + " does not exists");
}
- updatedProj = projectService.updateProject(projectDesc, currentProject);
+ if (projectDesc.getName().equals(currentProject.getName())) {
+ updatedProj = projectService.updateProject(projectDesc, currentProject);
+ } else {
+ updatedProj = projectService.renameProject(projectDesc, currentProject);
+ }
} catch (Exception e) {
logger.error("Failed to deal with the request.", e);
throw new InternalErrorException(e.getLocalizedMessage());
http://git-wip-us.apache.org/repos/asf/kylin/blob/450845c6/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
index 6dea4e3..a25e5b1 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
@@ -138,12 +138,12 @@ public class ProjectControllerV2 extends BasicController {
throw new BadRequestException(String.format(msg.getPROJECT_NOT_FOUND(), formerProjectName));
}
- // cannot modify project name if it's not empty
- if (!currentProject.getName().equals(projectDesc.getName()) && !isProjectEmpty(currentProject.getName())) {
- throw new BadRequestException(msg.getRENAME_PROJECT_NOT_EMPTY());
+ ProjectInstance updatedProj;
+ if (projectDesc.getName().equals(currentProject.getName())) {
+ updatedProj = projectService.updateProject(projectDesc, currentProject);
+ } else {
+ updatedProj = projectService.renameProject(projectDesc, currentProject);
}
-
- ProjectInstance updatedProj = projectService.updateProject(projectDesc, currentProject);
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, updatedProj, "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/450845c6/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
index 9ed38bb..45c1a65 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
@@ -249,10 +249,6 @@ public class Message {
return "Cannot delete non-empty project";
}
- public String getRENAME_PROJECT_NOT_EMPTY() {
- return "Cannot rename non-empty project";
- }
-
// Table
public String getHIVE_TABLE_NOT_FOUND() {
return "Cannot find Hive table '%s'.";
http://git-wip-us.apache.org/repos/asf/kylin/blob/450845c6/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index f71097b..c6bd6cf 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -28,6 +28,7 @@ import javax.annotation.Nullable;
import org.apache.directory.api.util.Strings;
import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.draft.Draft;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -94,6 +95,10 @@ public class ProjectService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#currentProject, 'ADMINISTRATION') or hasPermission(#currentProject, 'MANAGEMENT')")
public ProjectInstance updateProject(ProjectInstance newProject, ProjectInstance currentProject) throws IOException {
+ if (!newProject.getName().equals(currentProject.getName())) {
+ return renameProject(newProject, currentProject);
+ }
+
String newProjectName = newProject.getName();
String newDescription = newProject.getDescription();
LinkedHashMap<String, String> overrideProps = newProject.getOverrideKylinProps();
@@ -101,10 +106,27 @@ public class ProjectService extends BasicService {
ProjectInstance updatedProject = getProjectManager().updateProject(currentProject, newProjectName, newDescription, overrideProps);
logger.debug("Project updated.");
-
return updatedProject;
}
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#currentProject, 'ADMINISTRATION')")
+ public ProjectInstance renameProject(ProjectInstance newProject, ProjectInstance currentProject) throws IOException {
+ String newProjectName = newProject.getName();
+ String newDescription = newProject.getDescription();
+ LinkedHashMap<String, String> overrideProps = newProject.getOverrideKylinProps();
+
+ // rename project but keep UUID, acl keeps the same
+ ProjectInstance renamedProject = getProjectManager().renameProject(currentProject, newProjectName, newDescription, overrideProps);
+
+ // rebind draft and project
+ for (Draft draft : getDraftManager().list(currentProject.getName())) {
+ draft.setProject(newProjectName);
+ }
+
+ logger.debug("Project rename.");
+ return renamedProject;
+ }
+
@PostFilter(Constant.ACCESS_POST_FILTER_READ)
public List<ProjectInstance> listProjects(final Integer limit, final Integer offset) {
List<ProjectInstance> projects = listAllProjects(limit, offset);
[43/50] kylin git commit: minor, deduplicate returned acl entry
Posted by li...@apache.org.
minor, deduplicate returned acl entry
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6f8a91c0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6f8a91c0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6f8a91c0
Branch: refs/heads/master
Commit: 6f8a91c0dde0cf09efe3ff5e99533baed50f85c1
Parents: f79fa38
Author: Roger Shi <ro...@hotmail.com>
Authored: Mon Jun 26 20:12:39 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Mon Jun 26 20:23:10 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/service/AccessService.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6f8a91c0/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index 8b96635..e5cd793 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -20,7 +20,9 @@ package org.apache.kylin.rest.service;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
+import com.google.common.collect.Sets;
import org.apache.kylin.common.persistence.AclEntity;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.rest.constant.Constant;
@@ -298,10 +300,14 @@ public class AccessService {
public List<AccessEntryResponse> generateAceResponses(Acl acl) {
List<AccessEntryResponse> result = new ArrayList<AccessEntryResponse>();
+ Set<Sid> sidSet = Sets.newHashSet();
while (acl != null) {
for (AccessControlEntry ace : acl.getEntries()) {
- result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
+ if (!sidSet.contains(ace.getSid())) {
+ result.add(new AccessEntryResponse(ace.getId(), ace.getSid(), ace.getPermission(), ace.isGranting()));
+ sidSet.add(ace.getSid());
+ }
}
acl = acl.getParentAcl();
}
[35/50] kylin git commit: #1276 loose global dict column limit
Posted by li...@apache.org.
#1276 loose global dict column limit
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e6f58d1b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e6f58d1b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e6f58d1b
Branch: refs/heads/master
Commit: e6f58d1b8129d3ff024e6cfc1462dccbb65a27d8
Parents: c6ade2f
Author: Roger Shi <ro...@hotmail.com>
Authored: Fri Jun 23 10:09:06 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 23 19:14:31 2017 +0800
----------------------------------------------------------------------
.../kylin/cube/model/validation/rule/DictionaryRule.java | 7 -------
.../cube/model/validation/rule/DictionaryRuleTest.java | 10 ++--------
2 files changed, 2 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6f58d1b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
index 8da3ca0..548586f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/DictionaryRule.java
@@ -29,7 +29,6 @@ import org.apache.kylin.cube.model.DictionaryDesc;
import org.apache.kylin.cube.model.validation.IValidatorRule;
import org.apache.kylin.cube.model.validation.ResultLevel;
import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.dict.GlobalDictionaryBuilder;
import org.apache.kylin.metadata.model.TblColRef;
/**
@@ -46,7 +45,6 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> {
static final String ERROR_REUSE_BUILDER_BOTH_SET = "REUSE and BUILDER both set on dictionary for column: ";
static final String ERROR_REUSE_BUILDER_BOTH_EMPTY = "REUSE and BUILDER both empty on dictionary for column: ";
static final String ERROR_TRANSITIVE_REUSE = "Transitive REUSE is not allowed for dictionary: ";
- static final String ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE = "Global dictionary couldn't be used for dimension column: ";
@Override
public void validate(CubeDesc cubeDesc, ValidateContext context) {
@@ -82,11 +80,6 @@ public class DictionaryRule implements IValidatorRule<CubeDesc> {
return;
}
- if (StringUtils.isNotEmpty(builderClass) && builderClass.equalsIgnoreCase(GlobalDictionaryBuilder.class.getName()) && dimensionColumns.contains(dictCol)) {
- context.addResult(ResultLevel.ERROR, ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE + dictCol);
- return;
- }
-
if (reuseCol != null) {
reuseDictionaries.add(dictDesc);
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e6f58d1b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
index 0dd9b76..0f11f4b 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/model/validation/rule/DictionaryRuleTest.java
@@ -19,7 +19,6 @@
package org.apache.kylin.cube.model.validation.rule;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_DUPLICATE_DICTIONARY_COLUMN;
-import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_EMPTY;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_REUSE_BUILDER_BOTH_SET;
import static org.apache.kylin.cube.model.validation.rule.DictionaryRule.ERROR_TRANSITIVE_REUSE;
@@ -30,7 +29,6 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.util.List;
-import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
@@ -42,6 +40,8 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import com.google.common.collect.Lists;
+
public class DictionaryRuleTest extends LocalFileMetadataTestCase {
private static KylinConfig config;
@@ -96,12 +96,6 @@ public class DictionaryRuleTest extends LocalFileMetadataTestCase {
}
@Test
- public void testBadDesc5() throws IOException {
- testDictionaryDesc(ERROR_GLOBAL_DICTIONNARY_ONLY_MEASURE,
- DictionaryDesc.create("CATEG_LVL2_NAME", null, GlobalDictionaryBuilder.class.getName()));
- }
-
- @Test
public void testGoodDesc2() throws IOException {
testDictionaryDesc(null, DictionaryDesc.create("SELLER_ID", null, GlobalDictionaryBuilder.class.getName()));
}
[20/50] kylin git commit: #754 Change broadcast threads to daemon
threads
Posted by li...@apache.org.
#754 Change broadcast threads to daemon threads
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5a4e465d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5a4e465d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5a4e465d
Branch: refs/heads/master
Commit: 5a4e465dd8b8fda640db74e595491a2b2dd3c9db
Parents: 4d2a548
Author: auphyroc99 <45...@qq.com>
Authored: Tue Jun 20 14:44:01 2017 +0800
Committer: Roger Shi <ro...@gmail.com>
Committed: Tue Jun 20 15:01:11 2017 +0800
----------------------------------------------------------------------
.../kylin/metadata/cachesync/Broadcaster.java | 22 +++++++++++++-------
1 file changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5a4e465d/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 4fbfc7c..c9e1130 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -114,26 +114,28 @@ public class Broadcaster {
@Override
public void run() {
final Map<String, RestClient> restClientMap = Maps.newHashMap();
- final ExecutorService wipingCachePool = new ThreadPoolExecutor(1, 10, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
+ final ExecutorService wipingCachePool = new ThreadPoolExecutor(1, 10, 60L, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>(), new DaemonThreadFactory());
while (true) {
try {
final BroadcastEvent broadcastEvent = broadcastEvents.takeFirst();
String[] restServers = config.getRestServers();
- logger.info("Servers in the cluster: " + Arrays.toString(restServers));
+ logger.debug("Servers in the cluster: " + Arrays.toString(restServers));
for (final String node : restServers) {
if (restClientMap.containsKey(node) == false) {
restClientMap.put(node, new RestClient(node));
}
}
- logger.info("Announcing new broadcast event: " + broadcastEvent);
+ logger.debug("Announcing new broadcast event: " + broadcastEvent);
for (final String node : restServers) {
wipingCachePool.execute(new Runnable() {
@Override
public void run() {
try {
- restClientMap.get(node).wipeCache(broadcastEvent.getEntity(), broadcastEvent.getEvent(), broadcastEvent.getCacheKey());
+ restClientMap.get(node).wipeCache(broadcastEvent.getEntity(),
+ broadcastEvent.getEvent(), broadcastEvent.getCacheKey());
} catch (IOException e) {
logger.warn("Thread failed during wipe cache at " + broadcastEvent, e);
}
@@ -192,7 +194,8 @@ public class Broadcaster {
if (list == null)
return;
- logger.trace("Broadcasting metadata change: entity=" + entity + ", event=" + event + ", cacheKey=" + cacheKey + ", listeners=" + list);
+ logger.trace("Broadcasting metadata change: entity=" + entity + ", event=" + event + ", cacheKey=" + cacheKey
+ + ", listeners=" + list);
// prevents concurrent modification exception
list = Lists.newArrayList(list);
@@ -222,7 +225,8 @@ public class Broadcaster {
break;
}
- logger.debug("Done broadcasting metadata change: entity=" + entity + ", event=" + event + ", cacheKey=" + cacheKey);
+ logger.debug(
+ "Done broadcasting metadata change: entity=" + entity + ", event=" + event + ", cacheKey=" + cacheKey);
}
/**
@@ -279,7 +283,8 @@ public class Broadcaster {
public void onProjectDataChange(Broadcaster broadcaster, String project) throws IOException {
}
- public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException {
+ public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
+ throws IOException {
}
}
@@ -343,7 +348,8 @@ public class Broadcaster {
@Override
public String toString() {
- return Objects.toStringHelper(this).add("entity", entity).add("event", event).add("cacheKey", cacheKey).toString();
+ return Objects.toStringHelper(this).add("entity", entity).add("event", event).add("cacheKey", cacheKey)
+ .toString();
}
}
[37/50] kylin git commit: minor,
remove refine user cache in spring security
Posted by li...@apache.org.
minor, remove refine user cache in spring security
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3280172b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3280172b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3280172b
Branch: refs/heads/master
Commit: 3280172b737f85479eb3432fa12c267b477674fd
Parents: 22d8fae
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Jun 23 19:08:41 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 23 20:26:46 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 10 ++-
.../security/KylinAuthenticationProvider.java | 91 ++++++++++++--------
.../apache/kylin/rest/security/ManagedUser.java | 78 ++++++++---------
.../apache/kylin/rest/service/UserService.java | 10 +++
server/src/main/resources/ehcache-test.xml | 9 +-
server/src/main/resources/ehcache.xml | 9 +-
6 files changed, 111 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3280172b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2ccf3cf..ecd5261 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -502,7 +502,7 @@ abstract public class KylinConfigBase implements Serializable {
public Integer getSchedulerPollIntervalSecond() {
return Integer.parseInt(getOptional("kylin.job.scheduler.poll-interval-second", "30"));
}
-
+
public Integer getErrorRecordThreshold() {
return Integer.parseInt(getOptional("kylin.job.error-record-threshold", "0"));
}
@@ -1073,6 +1073,14 @@ abstract public class KylinConfigBase implements Serializable {
return getOptionalIntArray("kylin.server.query-metrics-percentiles-intervals", dft);
}
+ public int getServerUserCacheExpireSeconds() {
+ return Integer.valueOf(this.getOptional("kylin.server.auth-user-cache.expire-seconds", "300"));
+ }
+
+ public int getServerUserCacheMaxEntries() {
+ return Integer.valueOf(this.getOptional("kylin.server.auth-user-cache.max-entries", "100"));
+ }
+
// ============================================================================
// WEB
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/3280172b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
index 7322b84..dd9cbad 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/KylinAuthenticationProvider.java
@@ -18,7 +18,10 @@
package org.apache.kylin.rest.security;
-import org.apache.kylin.common.util.ByteArray;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.UserService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,13 +35,12 @@ import org.springframework.security.core.userdetails.UserDetails;
import org.springframework.security.core.userdetails.UsernameNotFoundException;
import org.springframework.util.Assert;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
-import net.sf.ehcache.Element;
-
/**
* A wrapper class for the authentication provider; Will do something more for Kylin.
*/
@@ -46,13 +48,21 @@ public class KylinAuthenticationProvider implements AuthenticationProvider {
private static final Logger logger = LoggerFactory.getLogger(KylinAuthenticationProvider.class);
+ private final static com.google.common.cache.Cache<String, Authentication> userCache = CacheBuilder.newBuilder()
+ .maximumSize(KylinConfig.getInstanceFromEnv().getServerUserCacheMaxEntries())
+ .expireAfterWrite(KylinConfig.getInstanceFromEnv().getServerUserCacheExpireSeconds(), TimeUnit.SECONDS)
+ .removalListener(new RemovalListener<String, Authentication>() {
+ @Override
+ public void onRemoval(RemovalNotification<String, Authentication> notification) {
+ KylinAuthenticationProvider.logger.debug("User cache {} is removed due to {}",
+ notification.getKey(), notification.getCause());
+ }
+ }).build();
+
@Autowired
@Qualifier("userService")
UserService userService;
- @Autowired
- private CacheManager cacheManager;
-
//Embedded authentication provider
private AuthenticationProvider authenticationProvider;
@@ -67,48 +77,53 @@ public class KylinAuthenticationProvider implements AuthenticationProvider {
@Override
public Authentication authenticate(Authentication authentication) throws AuthenticationException {
- Authentication authed = null;
- Cache userCache = cacheManager.getCache("UserCache");
+
byte[] hashKey = hf.hashString(authentication.getName() + authentication.getCredentials()).asBytes();
- ByteArray userKey = new ByteArray(hashKey);
+ String userKey = Arrays.toString(hashKey);
+
+ if (userService.isEvictCacheFlag()) {
+ userCache.invalidateAll();
+ userService.setEvictCacheFlag(false);
+ }
+ Authentication authed = userCache.getIfPresent(userKey);
- Element authedUser = userCache.get(userKey);
- if (null != authedUser) {
- authed = (Authentication) authedUser.getObjectValue();
+ if (null != authed) {
SecurityContextHolder.getContext().setAuthentication(authed);
} else {
try {
authed = authenticationProvider.authenticate(authentication);
- userCache.put(new Element(userKey, authed));
+
+ ManagedUser user;
+
+ if (authed.getDetails() == null) {
+ //authed.setAuthenticated(false);
+ throw new UsernameNotFoundException(
+ "User not found in LDAP, check whether he/she has been added to the groups.");
+ }
+
+ if (authed.getDetails() instanceof UserDetails) {
+ UserDetails details = (UserDetails) authed.getDetails();
+ user = new ManagedUser(details.getUsername(), details.getPassword(), false,
+ details.getAuthorities());
+ } else {
+ user = new ManagedUser(authentication.getName(), "skippped-ldap", false, authed.getAuthorities());
+ }
+ Assert.notNull(user, "The UserDetail is null.");
+
+ logger.debug("User {} authorities : {}", user.getUsername(), user.getAuthorities());
+ if (!userService.userExists(user.getUsername())) {
+ userService.createUser(user);
+ } else {
+ userService.updateUser(user);
+ }
+
+ userCache.put(userKey, authed);
} catch (AuthenticationException e) {
logger.error("Failed to auth user: " + authentication.getName(), e);
throw e;
}
logger.debug("Authenticated user " + authed.toString());
-
- ManagedUser user;
-
- if (authed.getDetails() == null) {
- //authed.setAuthenticated(false);
- throw new UsernameNotFoundException(
- "User not found in LDAP, check whether he/she has been added to the groups.");
- }
-
- if (authed.getDetails() instanceof UserDetails) {
- UserDetails details = (UserDetails) authed.getDetails();
- user = new ManagedUser(details.getUsername(), details.getPassword(), false, details.getAuthorities());
- } else {
- user = new ManagedUser(authentication.getName(), "skippped-ldap", false, authed.getAuthorities());
- }
- Assert.notNull(user, "The UserDetail is null.");
-
- logger.debug("User authorities :" + user.getAuthorities());
- if (!userService.userExists(user.getUsername())) {
- userService.createUser(user);
- } else {
- userService.updateUser(user);
- }
}
return authed;
http://git-wip-us.apache.org/repos/asf/kylin/blob/3280172b/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java b/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
index 4805d5c..280339e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/ManagedUser.java
@@ -22,8 +22,6 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.List;
-import javax.annotation.Nullable;
-
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.rest.service.UserGrantedAuthority;
import org.springframework.security.core.GrantedAuthority;
@@ -31,8 +29,6 @@ import org.springframework.security.core.userdetails.UserDetails;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
@SuppressWarnings("serial")
@@ -44,7 +40,7 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
@JsonProperty
private String password;
@JsonProperty
- private List<String> authorities = Lists.newArrayList();
+ private List<UserGrantedAuthority> authorities = Lists.newArrayList();
@JsonProperty
private boolean disabled = false;
@JsonProperty
@@ -56,24 +52,40 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
@JsonProperty
private int wrongTime = 0;
- private Boolean legacyCatered = false;
//DISABLED_ROLE is a ancient way to represent disabled user
//now we no longer support such way, however legacy metadata may still contain it
private static final String DISABLED_ROLE = "--disabled--";
- //this is computed
- private List<UserGrantedAuthority> grantedAuthorities = null;
-
public ManagedUser() {
}
- public ManagedUser(String username, String password, Boolean defaultPassword, String... authorities) {
+ public ManagedUser(@JsonProperty String username, @JsonProperty String password,
+ @JsonProperty List<UserGrantedAuthority> authorities, @JsonProperty boolean disabled,
+ @JsonProperty boolean defaultPassword, @JsonProperty boolean locked, @JsonProperty long lockedTime,
+ @JsonProperty int wrongTime) {
+ this.username = username;
+ this.password = password;
+ this.authorities = authorities;
+ this.disabled = disabled;
+ this.defaultPassword = defaultPassword;
+ this.locked = locked;
+ this.lockedTime = lockedTime;
+ this.wrongTime = wrongTime;
+
+ caterLegacy();
+ }
+
+ public ManagedUser(String username, String password, Boolean defaultPassword, String... authoritiesStr) {
this.username = username;
this.password = password;
this.setDefaultPassword(defaultPassword);
- this.authorities = Lists.newArrayList(authorities);
- this.grantedAuthorities = null;
+ this.authorities = Lists.newArrayList();
+ for (String a : authoritiesStr) {
+ authorities.add(new UserGrantedAuthority(a));
+ }
+
+ caterLegacy();
}
public ManagedUser(String username, String password, Boolean defaultPassword,
@@ -83,6 +95,8 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
this.setDefaultPassword(defaultPassword);
this.setGrantedAuthorities(grantedAuthorities);
+
+ caterLegacy();
}
public String getUsername() {
@@ -102,45 +116,27 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
}
private void caterLegacy() {
- if (!legacyCatered) {
- synchronized (legacyCatered) {
- Iterator<String> iterator = authorities.iterator();
- while (iterator.hasNext()) {
- if (DISABLED_ROLE.equals(iterator.next())) {
- iterator.remove();
- this.disabled = true;
- }
- }
- legacyCatered = true;
+ Iterator<UserGrantedAuthority> iterator = authorities.iterator();
+ while (iterator.hasNext()) {
+ if (DISABLED_ROLE.equals(iterator.next().getAuthority())) {
+ iterator.remove();
+ this.disabled = true;
}
}
}
public List<UserGrantedAuthority> getAuthorities() {
- caterLegacy();
- if (grantedAuthorities == null) {
- grantedAuthorities = Lists.newArrayList();
- for (String a : authorities) {
- this.grantedAuthorities.add(new UserGrantedAuthority(a));
- }
- }
- return grantedAuthorities;
+ return this.authorities;
}
public void setGrantedAuthorities(Collection<? extends GrantedAuthority> grantedAuthorities) {
- this.authorities = Lists
- .newArrayList(Collections2.transform(grantedAuthorities, new Function<GrantedAuthority, String>() {
- @Nullable
- @Override
- public String apply(@Nullable GrantedAuthority input) {
- return input.getAuthority();
- }
- }));
- this.grantedAuthorities = null;
+ this.authorities = Lists.newArrayList();
+ for (GrantedAuthority grantedAuthority : grantedAuthorities) {
+ this.authorities.add(new UserGrantedAuthority(grantedAuthority.getAuthority()));
+ }
}
public boolean isDisabled() {
- caterLegacy();
return disabled;
}
@@ -230,6 +226,6 @@ public class ManagedUser extends RootPersistentEntity implements UserDetails {
@Override
public String toString() {
- return "KapManagedUser [username=" + username + ", authorities=" + grantedAuthorities + "]";
+ return "ManagedUser [username=" + username + ", authorities=" + authorities + "]";
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3280172b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 504c035..6682c03 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -54,6 +54,16 @@ public class UserService implements UserDetailsManager {
public static final Serializer<ManagedUser> SERIALIZER = new JsonSerializer<>(ManagedUser.class);
protected ResourceStore aclStore;
+
+ private boolean evictCacheFlag = false;
+
+ public boolean isEvictCacheFlag() {
+ return evictCacheFlag;
+ }
+
+ public void setEvictCacheFlag(boolean evictCacheFlag) {
+ this.evictCacheFlag = evictCacheFlag;
+ }
@PostConstruct
public void init() throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3280172b/server/src/main/resources/ehcache-test.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/ehcache-test.xml b/server/src/main/resources/ehcache-test.xml
index bffe27a..5bd4d13 100644
--- a/server/src/main/resources/ehcache-test.xml
+++ b/server/src/main/resources/ehcache-test.xml
@@ -27,11 +27,4 @@
>
<persistence strategy="none"/>
</cache>
- <cache name="UserCache"
- eternal="false"
- timeToLiveSeconds="10800"
- memoryStoreEvictionPolicy="LRU"
- >
- <persistence strategy="none"/>
- </cache>
-</ehcache>
\ No newline at end of file
+</ehcache>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3280172b/server/src/main/resources/ehcache.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/ehcache.xml b/server/src/main/resources/ehcache.xml
index 1d49ca6..c9efc13 100644
--- a/server/src/main/resources/ehcache.xml
+++ b/server/src/main/resources/ehcache.xml
@@ -27,11 +27,4 @@
>
<persistence strategy="none"/>
</cache>
- <cache name="UserCache"
- eternal="false"
- timeToLiveSeconds="10800"
- memoryStoreEvictionPolicy="LRU"
- >
- <persistence strategy="none"/>
- </cache>
-</ehcache>
\ No newline at end of file
+</ehcache>
[40/50] kylin git commit: minor, fix thread local KylinConfig bug
Posted by li...@apache.org.
minor, fix thread local KylinConfig bug
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0f8c21ba
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0f8c21ba
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0f8c21ba
Branch: refs/heads/master
Commit: 0f8c21ba350afee829a5e847d234705d8b2ef559
Parents: 81ff9a4
Author: Hongbin Ma <ma...@apache.org>
Authored: Sun Jun 25 17:03:16 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Sun Jun 25 17:17:49 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/common/KylinConfig.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0f8c21ba/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 0f77096..1298807 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -57,7 +57,7 @@ public class KylinConfig extends KylinConfigBase {
private static KylinConfig SYS_ENV_INSTANCE = null;
// thread-local instances, will override SYS_ENV_INSTANCE
- private static final transient ThreadLocal<KylinConfig> THREAD_ENV_INSTANCE = new ThreadLocal<>();
+ private static transient ThreadLocal<KylinConfig> THREAD_ENV_INSTANCE = new ThreadLocal<>();
public static KylinConfig getInstanceFromEnv() {
synchronized (KylinConfig.class) {
@@ -88,7 +88,7 @@ public class KylinConfig extends KylinConfigBase {
logger.info("Destroy KylinConfig");
dumpStackTrace();
SYS_ENV_INSTANCE = null;
- THREAD_ENV_INSTANCE.remove();
+ THREAD_ENV_INSTANCE = new ThreadLocal<>();
}
}
@@ -197,6 +197,10 @@ public class KylinConfig extends KylinConfigBase {
public static void setKylinConfigThreadLocal(KylinConfig config) {
THREAD_ENV_INSTANCE.set(config);
}
+
+ public static void removeKylinConfigThreadLocal() {
+ THREAD_ENV_INSTANCE.remove();
+ }
public static KylinConfig createKylinConfig(String propsInStr) throws IOException {
Properties props = new Properties();
[07/50] kylin git commit: minor, improve odbc performance
Posted by li...@apache.org.
minor, improve odbc performance
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/51f22e28
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/51f22e28
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/51f22e28
Branch: refs/heads/master
Commit: 51f22e284467f23c8aeeed8ea79e5133ddd1141e
Parents: f32165e
Author: lidongsjtu <li...@apache.org>
Authored: Thu Jun 15 17:03:20 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 15 21:17:54 2017 +0800
----------------------------------------------------------------------
odbc/Common/REST.cpp | 67 +++++++++++++++++++++--------------
odbc/Common/REST.h | 1 +
odbc/Driver/KO_DIAG.CPP | 17 ++++-----
odbc/Driver/KO_EXEC.CPP | 6 ++--
odbc/TestDLL/SimpleQueryTest.cpp | 6 ++--
5 files changed, 57 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Common/REST.cpp
----------------------------------------------------------------------
diff --git a/odbc/Common/REST.cpp b/odbc/Common/REST.cpp
index 858ecd4..8f0c7bb 100644
--- a/odbc/Common/REST.cpp
+++ b/odbc/Common/REST.cpp
@@ -280,34 +280,28 @@ std::unique_ptr <MetadataResponse> restGetMeta ( char* serverAddr, long port, ch
wstring cookQuery ( wchar_t* p )
{
- wchar_t* q = new wchar_t[wcslen ( p ) + 1];
- wcscpy ( q, p );
-
- for ( int i = 0; i < ( int ) wcslen ( q ); i++ )
- {
- if ( q[i] == '\r' || q[i] == '\n' || q[i] == '\t' )
- {
- q[i] = ' ';
- }
- }
-
- wstring ret ( q );
- delete[] q;
- size_t pos = 0;
+ std::wstringstream wss;
- for ( size_t pos = 0;; pos += 2 )
+ int l = wcslen ( p );
+ for ( int i = 0; i < l; i++ )
{
- pos = ret . find ( L"\"", pos );
-
- if ( pos == wstring::npos )
+ if ( p[i] == L'\r' || p[i] == L'\n' || p[i] == L'\t' )
{
- break;
- }
-
- ret . insert ( pos, L"\\" );
+ wss << L' ';
+ }
+
+ else if (p[i] == L'"')
+ {
+ wss << L"\\\"";
+ }
+
+ else
+ {
+ wss << p[i];
+ }
}
- return ret;
+ return wss.str();
}
wstring getBodyString ( http_response& response )
@@ -389,10 +383,15 @@ std::unique_ptr <SQLResponse> convertToSQLResponse ( int statusFlag,
wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* username,
char* passwd,
char* project,
+ bool isPrepare,
int* statusFlag)
{
//using local cache to intercept probing queries
- const wchar_t* cachedQueryRes = loadCache ( rawSql );
+ const wchar_t* cachedQueryRes = NULL;
+
+ if (isPrepare) {
+ cachedQueryRes = loadCache ( rawSql );
+ }
if ( cachedQueryRes != NULL )
{
@@ -404,8 +403,24 @@ wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* usern
wstring serverAddrW = completeServerStr ( serverAddr, port );
http_client_config config;
config . set_timeout ( utility::seconds ( 36000 ) );
+
+ //uncomment these lines for debug with proxy
+ //wstring p = L"http://127.0.0.1:8888";
+ //config.set_proxy(web_proxy(p));
+
http_client session ( serverAddrW, config );
- http_request request = makeRequest ( username, passwd, L"/kylin/api/query", methods::POST );
+ http_request request;
+
+ if (!isPrepare)
+ {
+ request = makeRequest ( username, passwd, L"/kylin/api/query", methods::POST );
+ }
+
+ else
+ {
+ request = makeRequest ( username, passwd, L"/kylin/api/query/prestate", methods::POST );
+ }
+
wstring sql = cookQuery ( rawSql );
std::wstringstream wss;
wss << L"{ \"acceptPartial\": false, \"project\" : \"" << project << L"\", " << " \"sql\" : \"" << sql << L"\" }" ;
@@ -444,7 +459,7 @@ wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* usern
wstring ret = getBodyString ( response );
- if (*statusFlag == 1)
+ if (*statusFlag == 1 && isPrepare)
{
storeCache(rawSql, ret.c_str());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Common/REST.h
----------------------------------------------------------------------
diff --git a/odbc/Common/REST.h b/odbc/Common/REST.h
index f70d7d5..60c14a7 100644
--- a/odbc/Common/REST.h
+++ b/odbc/Common/REST.h
@@ -38,4 +38,5 @@ std::unique_ptr <SQLResponse> convertToSQLResponse ( int status,
wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* username,
char* passwd,
char* project,
+ bool isPrepare,
int* statusFlag);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Driver/KO_DIAG.CPP
----------------------------------------------------------------------
diff --git a/odbc/Driver/KO_DIAG.CPP b/odbc/Driver/KO_DIAG.CPP
index ac313e7..def819c 100644
--- a/odbc/Driver/KO_DIAG.CPP
+++ b/odbc/Driver/KO_DIAG.CPP
@@ -644,7 +644,8 @@ RETCODE SQL_API SQLGetDiagRec ( SQLSMALLINT pHandleType,
pODBCDiagRow _SQLPutDiagRow ( pODBCDiag pDiag, StrPtr pFunc, StrPtr pState, Long pNativeErrorCode,
StrPtr pMsgArgs, va_list pArgs )
{
- Char s[4096]; // arbitary and maximum length of message
+ // to avoid: TST1003: The message for sqlstate '01000' is longer than SQL_MAX_MESSAGE_LENGTH - 1.
+ Char s[SQL_MAX_MESSAGE_LENGTH - 1]; // arbitary and maximum length of message
pODBCDiagRow l;
pODBCDiagRow r;
@@ -660,7 +661,7 @@ pODBCDiagRow _SQLPutDiagRow ( pODBCDiag pDiag, StrPtr pFunc, StrPtr pState, Long
// check if there is some message
if ( pMsgArgs )
{
- vsprintf ( s + strlen ( s ), pMsgArgs, pArgs );
+ vsnprintf ( s + strlen ( s ), SQL_MAX_MESSAGE_LENGTH - 2 - strlen(s), pMsgArgs, pArgs );
}
else
@@ -1110,13 +1111,13 @@ void _ODBCLogMsg ( LogLevel level, const char* pMsgArgs, ... )
break;
}
- time_t now = time ( 0 );
- struct tm tstruct;
- char buffer[100];
- tstruct = *localtime ( &now );
- strftime ( buffer, 100, "%Y-%m-%d.%X", &tstruct );
+ char ts[24];
+ SYSTEMTIME sys;
+ GetLocalTime( &sys );
+ sprintf( ts, "%4d-%02d-%02d %02d:%02d:%02d.%03d", sys.wYear, sys.wMonth, sys.wDay, sys.wHour, sys.wMinute, sys.wSecond, sys.wMilliseconds);
+
_write ( gLogFile, "[", 1 );
- _write ( gLogFile, buffer, strlen ( buffer ) );
+ _write ( gLogFile, ts, strlen ( ts ) );
_write ( gLogFile, "]", 1 );
// MSG PARSING
http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Driver/KO_EXEC.CPP
----------------------------------------------------------------------
diff --git a/odbc/Driver/KO_EXEC.CPP b/odbc/Driver/KO_EXEC.CPP
index 514375e..8a8992c 100644
--- a/odbc/Driver/KO_EXEC.CPP
+++ b/odbc/Driver/KO_EXEC.CPP
@@ -129,7 +129,7 @@ RETCODE SQL_API _SQLExecStmtFromReq ( pODBCStmt pStmt, bool pPrepared ) {
try {
__ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "start to call rest api" ) );
response = requestQuery ( pStmt->Stmt, pStmt->Conn->Server, pStmt->Conn->ServerPort, pStmt->Conn->UserName, pStmt->Conn->Password,
- pStmt->Conn->Project, &status );
+ pStmt->Conn->Project, pPrepared, &status );
__ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "received and uncompressed rest response:" ) );
p = convertToSQLResponse(status, response);
__ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "parsed to SQLResponse" ) );
@@ -207,7 +207,7 @@ RETCODE SQL_API SQLPrepareW ( SQLHSTMT pStmt,
( ( pODBCStmt ) pStmt )->Prepared = 1;
- return _SQLExecStmtFromReq((pODBCStmt)pStmt,1);
+ return _SQLExecStmtFromReq((pODBCStmt)pStmt, 1);
//return SQL_SUCCESS;
}
@@ -230,7 +230,7 @@ RETCODE SQL_API SQLExecute ( HSTMT pStmt ) {
}
// excute the request
- x = _SQLExecStmtFromReq ( ( pODBCStmt ) pStmt, 1 );
+ x = _SQLExecStmtFromReq ( ( pODBCStmt ) pStmt, 0 );
return x;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/TestDLL/SimpleQueryTest.cpp
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/SimpleQueryTest.cpp b/odbc/TestDLL/SimpleQueryTest.cpp
index 7f857a9..9705b5e 100644
--- a/odbc/TestDLL/SimpleQueryTest.cpp
+++ b/odbc/TestDLL/SimpleQueryTest.cpp
@@ -23,7 +23,7 @@ void simpleQueryTest ()
//Intercept query test
{
int status;
- wstring s = requestQuery ( L"SELECT 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, &status );
+ wstring s = requestQuery ( L"SELECT 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, false, &status );
std::unique_ptr <SQLResponse> y = convertToSQLResponse(status, s);
if ( ( int ) y -> results . size () != 1 )
@@ -34,7 +34,7 @@ void simpleQueryTest ()
//Ungzipped Query Test
{
int status;
- wstring s = requestQuery ( L"select cal_dt from test_kylin_fact limit 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, &status );
+ wstring s = requestQuery ( L"select cal_dt from test_kylin_fact limit 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, false, &status );
std::unique_ptr <SQLResponse> y = convertToSQLResponse(status, s);
if ( ( int ) y -> results . size () != 1 )
@@ -45,7 +45,7 @@ void simpleQueryTest ()
//zipped Query Test
{
int status;
- wstring s = requestQuery ( L"select * from test_kylin_fact limit 12", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, &status );
+ wstring s = requestQuery ( L"select * from test_kylin_fact limit 12", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, false, &status );
std::unique_ptr <SQLResponse> y = convertToSQLResponse(status, s);
if ( ( int ) y -> results . size () != 12 )
[38/50] kylin git commit: minor, fix NPE when limit or offset is null
Posted by li...@apache.org.
minor, fix NPE when limit or offset is null
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ba7bfd61
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ba7bfd61
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ba7bfd61
Branch: refs/heads/master
Commit: ba7bfd6131f3bb296dacbec3066340ffdce5765b
Parents: 3280172
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Jun 23 21:04:28 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Fri Jun 23 21:26:05 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/request/SQLRequest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ba7bfd61/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
index 4b3d9c8..54900ba 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java
@@ -68,7 +68,7 @@ public class SQLRequest implements Serializable {
}
public Integer getOffset() {
- return offset;
+ return offset == null ? 0 : offset;
}
public void setOffset(Integer offset) {
@@ -76,7 +76,7 @@ public class SQLRequest implements Serializable {
}
public Integer getLimit() {
- return limit;
+ return limit == null ? 0 : limit;
}
public void setLimit(Integer limit) {
[11/50] kylin git commit: Support schedule streaming cube (#1148)
Posted by li...@apache.org.
Support schedule streaming cube (#1148)
* Support schedule streaming cube
* Resume schedulers after login
* Set streaming cube's endOffset from Integer.MAX_VALUE to Long.MAX_VALUE
* Call AccessService apis only for rest apis.
* Resume scheduler jobs after scheduler service initialized.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81d26370
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81d26370
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81d26370
Branch: refs/heads/master
Commit: 81d26370c8365faf1de2c74cfb2af7869d786a1a
Parents: 6182589
Author: Ni Chunen <zj...@sjtu.org>
Authored: Fri Jun 16 21:24:14 2017 +0800
Committer: Roger Shi <ro...@gmail.com>
Committed: Fri Jun 16 21:24:14 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/service/JobService.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/81d26370/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 7eb1292..6451d66 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -205,8 +205,13 @@ public class JobService extends BasicService implements InitializingBean {
public JobInstance submitJob(CubeInstance cube, long startDate, long endDate, long startOffset, long endOffset, //
Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd,
CubeBuildTypeEnum buildType, boolean force, String submitter) throws IOException {
- return submitJobInternal(cube, startDate, endDate, startOffset, endOffset, sourcePartitionOffsetStart,
+ JobInstance jobInstance = submitJobInternal(cube, startDate, endDate, startOffset, endOffset, sourcePartitionOffsetStart,
sourcePartitionOffsetEnd, buildType, force, submitter);
+
+ accessService.init(jobInstance, null);
+ accessService.inherit(jobInstance, cube);
+
+ return jobInstance;
}
public JobInstance submitJobInternal(CubeInstance cube, long startDate, long endDate, long startOffset,
@@ -262,9 +267,6 @@ public class JobService extends BasicService implements InitializingBean {
JobInstance jobInstance = getSingleJobInstance(job);
- accessService.init(jobInstance, null);
- accessService.inherit(jobInstance, cube);
-
return jobInstance;
}
[46/50] kylin git commit: KYLIN-2681 Convert input sql's expression
to computed column if computed colum defined
Posted by li...@apache.org.
KYLIN-2681 Convert input sql's expression to computed column if computed colum defined
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/05631b58
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/05631b58
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/05631b58
Branch: refs/heads/master
Commit: 05631b588c0b49bb156e85dcaf3dca27e79b700c
Parents: 4f20ba3
Author: Aron.tao <24...@qq.com>
Authored: Sun Jun 25 18:45:27 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Tue Jun 27 18:29:11 2017 +0800
----------------------------------------------------------------------
.../query/util/CognosParenthesesEscape.java | 2 +-
.../query/util/ConvertToComputedColumn.java | 353 +++++++++++++++++++
.../query/util/KeywordDefaultDirtyHack.java | 2 +-
.../org/apache/kylin/query/util/QueryUtil.java | 11 +-
.../query/util/CognosParenthesesEscapeTest.java | 32 +-
.../query/util/ConvertToComputedColumnTest.java | 135 +++++++
.../apache/kylin/query/util/QueryUtilTest.java | 6 +-
.../apache/kylin/rest/service/QueryService.java | 2 +-
8 files changed, 520 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/05631b58/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java b/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
index 6d930a5..8c6d82d 100644
--- a/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
+++ b/query/src/main/java/org/apache/kylin/query/util/CognosParenthesesEscape.java
@@ -32,7 +32,7 @@ public class CognosParenthesesEscape implements QueryUtil.IQueryTransformer {
private static final Pattern FROM_PATTERN = Pattern.compile("\\s+from\\s+(\\s*\\(\\s*)+(?!\\s*select\\s)", Pattern.CASE_INSENSITIVE);
@Override
- public String transform(String sql) {
+ public String transform(String sql, String project) {
if (sql == null || sql.isEmpty()) {
return sql;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/05631b58/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
new file mode 100644
index 0000000..d8f1220
--- /dev/null
+++ b/query/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.util;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import javax.annotation.Nullable;
+
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlDataTypeSpec;
+import org.apache.calcite.sql.SqlDynamicParam;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlLiteral;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.util.SqlVisitor;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Functions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Ordering;
+
+public class ConvertToComputedColumn implements QueryUtil.IQueryTransformer {
+ private static final Logger logger = LoggerFactory.getLogger(ConvertToComputedColumn.class);
+
+ @Override
+ public String transform(String sql, String project) {
+ if (project == null) {
+ return sql;
+ }
+ ImmutableSortedMap<String, String> computedColumns = getSortedComputedColumnWithProject(project);
+ return replaceComputedColumn(sql, computedColumns);
+ }
+
+ static String replaceComputedColumn(String inputSql, ImmutableSortedMap<String, String> computedColumn) {
+ if (inputSql == null) {
+ return "";
+ }
+
+ if (computedColumn == null || computedColumn.isEmpty()) {
+ return inputSql;
+ }
+ String result = inputSql;
+ String[] lines = inputSql.split("\n");
+ List<Pair<String, String>> toBeReplacedExp = new ArrayList<>(); //{"alias":"expression"}, like {"t1":"t1.a+t1.b+t1.c"}
+
+ for (String ccExp : computedColumn.keySet()) {
+ List<SqlNode> matchedNodes = new ArrayList<>();
+ try {
+ matchedNodes = getMatchedNodes(inputSql, computedColumn.get(ccExp));
+ } catch (SqlParseException e) {
+ logger.error("Convert to computedColumn Fail,parse sql fail ", e.getMessage());
+ }
+ for (SqlNode node : matchedNodes) {
+ Pair<Integer, Integer> startEndPos = getReplacePos(lines, node);
+ int start = startEndPos.getLeft();
+ int end = startEndPos.getRight();
+ //add table alias like t1.column,if exists alias
+ String alias = getTableAlias(node);
+ toBeReplacedExp.add(Pair.of(alias, inputSql.substring(start, end)));
+ }
+ logger.debug("Computed column: " + ccExp + "'s matched list:" + toBeReplacedExp);
+ //replace user's input sql
+ for (Pair<String, String> toBeReplaced : toBeReplacedExp) {
+ result = result.replace(toBeReplaced.getRight(), toBeReplaced.getLeft() + ccExp);
+ }
+ }
+ return result;
+ }
+
+ private static Pair<Integer, Integer> getReplacePos(String[] lines, SqlNode node) {
+ SqlParserPos pos = node.getParserPosition();
+ int lineStart = pos.getLineNum();
+ int columnStart = pos.getColumnNum() - 1;
+ int columnEnd = pos.getEndColumnNum();
+ //for the case that sql is multi lines
+ for (int i = 0; i < lineStart - 1; i++) {
+ int offset = lines[i].length();
+ columnStart += offset + 1;
+ columnEnd += offset + 1;
+ }
+ return Pair.of(columnStart, columnEnd);
+ }
+
+ //Return matched node's position and its alias(if exists).If can not find matches, return an empty capacity list
+ private static List<SqlNode> getMatchedNodes(String inputSql, String ccExp) throws SqlParseException {
+ if (ccExp == null || ccExp.equals("")) {
+ return new ArrayList<>();
+ }
+ ArrayList<SqlNode> toBeReplacedNodes = new ArrayList<>();
+ SqlNode ccNode = getCCExpNode(ccExp);
+ List<SqlNode> inputNodes = getInputTreeNodes(inputSql);
+
+ // find whether user input sql's tree node equals computed columns's define expression
+ for (SqlNode inputNode : inputNodes) {
+ if (isNodeEqual(inputNode, ccNode)) {
+ toBeReplacedNodes.add(inputNode);
+ }
+ }
+ return toBeReplacedNodes;
+ }
+
+ private static List<SqlNode> getInputTreeNodes(String inputSql) throws SqlParseException {
+ SqlTreeVisitor stv = new SqlTreeVisitor();
+ parse(inputSql).accept(stv);
+ return stv.getSqlNodes();
+ }
+
+ private static SqlNode getCCExpNode(String ccExp) throws SqlParseException {
+ ccExp = "select " + ccExp + " from t";
+ return ((SqlSelect) parse(ccExp)).getSelectList().get(0);
+ }
+
+ static SqlNode parse(String sql) throws SqlParseException {
+ SqlParser.ConfigBuilder parserBuilder = SqlParser.configBuilder();
+ SqlParser sqlParser = SqlParser.create(sql, parserBuilder.build());
+ return sqlParser.parseQuery();
+ }
+
+ static boolean isNodeEqual(SqlNode node0, SqlNode node1) {
+ if (node0 == null) {
+ return node1 == null;
+ } else if (node1 == null) {
+ return false;
+ }
+
+ if (!Objects.equals(node0.getClass().getSimpleName(), node1.getClass().getSimpleName())) {
+ return false;
+ }
+
+ if (node0 instanceof SqlCall) {
+ SqlCall thisNode = (SqlCall) node0;
+ SqlCall thatNode = (SqlCall) node1;
+ if (!thisNode.getOperator().getName().equalsIgnoreCase(thatNode.getOperator().getName())) {
+ return false;
+ }
+ return isNodeEqual(thisNode.getOperandList(), thatNode.getOperandList());
+ }
+ if (node0 instanceof SqlLiteral) {
+ SqlLiteral thisNode = (SqlLiteral) node0;
+ SqlLiteral thatNode = (SqlLiteral) node1;
+ return Objects.equals(thisNode.getValue(), thatNode.getValue());
+ }
+ if (node0 instanceof SqlNodeList) {
+ SqlNodeList thisNode = (SqlNodeList) node0;
+ SqlNodeList thatNode = (SqlNodeList) node1;
+ if (thisNode.getList().size() != thatNode.getList().size()) {
+ return false;
+ }
+ for (int i = 0; i < thisNode.getList().size(); i++) {
+ SqlNode thisChild = thisNode.getList().get(i);
+ final SqlNode thatChild = thatNode.getList().get(i);
+ if (!isNodeEqual(thisChild, thatChild)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ if (node0 instanceof SqlIdentifier) {
+ SqlIdentifier thisNode = (SqlIdentifier) node0;
+ SqlIdentifier thatNode = (SqlIdentifier) node1;
+ // compare ignore table alias.eg: expression like "a.b + a.c + a.d" ,alias a will be ignored when compared
+ String name0 = thisNode.names.get(thisNode.names.size() - 1).replace("\"", "");
+ String name1 = thatNode.names.get(thatNode.names.size() - 1).replace("\"", "");
+ return name0.equalsIgnoreCase(name1);
+ }
+
+ logger.error("Convert to computed column fail,failed to compare two nodes,unknown instance type");
+ return false;
+ }
+
+ private static boolean isNodeEqual(List<SqlNode> operands0, List<SqlNode> operands1) {
+ if (operands0.size() != operands1.size()) {
+ return false;
+ }
+ for (int i = 0; i < operands0.size(); i++) {
+ if (!isNodeEqual(operands0.get(i), operands1.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static String getTableAlias(SqlNode node) {
+ if (node instanceof SqlCall) {
+ SqlCall call = (SqlCall) node;
+ return getTableAlias(call.getOperandList());
+ }
+ if (node instanceof SqlIdentifier) {
+ StringBuilder alias = new StringBuilder("");
+ ImmutableList<String> names = ((SqlIdentifier) node).names;
+ if (names.size() >= 2) {
+ for (int i = 0; i < names.size() - 1; i++) {
+ alias.append(names.get(i)).append(".");
+ }
+ }
+ return alias.toString();
+ }
+ if (node instanceof SqlNodeList) {
+ return "";
+ }
+ if (node instanceof SqlLiteral) {
+ return "";
+ }
+ return "";
+ }
+
+ private static String getTableAlias(List<SqlNode> operands) {
+ for (SqlNode operand : operands) {
+ return getTableAlias(operand);
+ }
+ return "";
+ }
+
+ private ImmutableSortedMap<String, String> getSortedComputedColumnWithProject(String project) {
+ MetadataManager metadataManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ Map<String, MetadataManager.CCInfo> ccInfoMap = metadataManager.getCcInfoMap();
+ final ProjectInstance projectInstance = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
+ .getProject(project);
+
+ Iterable<MetadataManager.CCInfo> projectCCInfo = Iterables.filter(ccInfoMap.values(),
+ new Predicate<MetadataManager.CCInfo>() {
+ @Override
+ public boolean apply(@Nullable MetadataManager.CCInfo ccInfo) {
+ return Iterables.any(ccInfo.getDataModelDescs(), new Predicate<DataModelDesc>() {
+ @Override
+ public boolean apply(@Nullable DataModelDesc model) {
+ return projectInstance.containsModel(model.getName());
+ }
+ });
+ }
+ });
+
+ Map<String, String> computedColumns = new HashMap<>();
+ for (MetadataManager.CCInfo ccInfo : projectCCInfo) {
+ computedColumns.put(ccInfo.getComputedColumnDesc().getColumnName(),
+ ccInfo.getComputedColumnDesc().getExpression());
+ }
+
+ return getMapSortedByValue(computedColumns);
+ }
+
+ static ImmutableSortedMap<String, String> getMapSortedByValue(Map<String, String> computedColumns) {
+ if (computedColumns == null || computedColumns.isEmpty()) {
+ return null;
+ }
+
+ Ordering<String> ordering = Ordering.from(new Comparator<String>() {
+ @Override
+ public int compare(String o1, String o2) {
+ return Integer.compare(o1.replaceAll("\\s*", "").length(), o2.replaceAll("\\s*", "").length());
+ }
+ }).reverse().nullsLast().onResultOf(Functions.forMap(computedColumns, null)).compound(Ordering.natural());
+ return ImmutableSortedMap.copyOf(computedColumns, ordering);
+ }
+
+}
+
+class SqlTreeVisitor implements SqlVisitor<SqlNode> {
+ private List<SqlNode> sqlNodes;
+
+ SqlTreeVisitor() {
+ this.sqlNodes = new ArrayList<>();
+ }
+
+ List<SqlNode> getSqlNodes() {
+ return sqlNodes;
+ }
+
+ @Override
+ public SqlNode visit(SqlNodeList nodeList) {
+ sqlNodes.add(nodeList);
+ for (int i = 0; i < nodeList.size(); i++) {
+ SqlNode node = nodeList.get(i);
+ node.accept(this);
+ }
+ return null;
+ }
+
+ @Override
+ public SqlNode visit(SqlLiteral literal) {
+ sqlNodes.add(literal);
+ return null;
+ }
+
+ @Override
+ public SqlNode visit(SqlCall call) {
+ sqlNodes.add(call);
+ for (SqlNode operand : call.getOperandList()) {
+ if (operand != null) {
+ operand.accept(this);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public SqlNode visit(SqlIdentifier id) {
+ sqlNodes.add(id);
+ return null;
+ }
+
+ @Override
+ public SqlNode visit(SqlDataTypeSpec type) {
+ return null;
+ }
+
+ @Override
+ public SqlNode visit(SqlDynamicParam param) {
+ return null;
+ }
+
+ @Override
+ public SqlNode visit(SqlIntervalQualifier intervalQualifier) {
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/05631b58/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java b/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
index e1398f6..23faf8e 100644
--- a/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
+++ b/query/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
@@ -21,7 +21,7 @@ package org.apache.kylin.query.util;
public class KeywordDefaultDirtyHack implements QueryUtil.IQueryTransformer {
@Override
- public String transform(String sql) {
+ public String transform(String sql, String project) {
// KYLIN-2108, DEFAULT is hive default database, but a sql keyword too, needs quote
sql = sql.replace("DEFAULT.", "\"DEFAULT\".");
sql = sql.replace("default.", "\"default\".");
http://git-wip-us.apache.org/repos/asf/kylin/blob/05631b58/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index d48a26f..7794f94 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -38,14 +38,15 @@ public class QueryUtil {
private static List<IQueryTransformer> queryTransformers;
public interface IQueryTransformer {
- String transform(String sql);
+ String transform(String sql, String project);
}
+ // for mockup test
public static String massageSql(String sql) {
- return massageSql(sql, 0, 0);
+ return massageSql(sql, null, 0, 0);
}
- public static String massageSql(String sql, int limit, int offset) {
+ public static String massageSql(String sql, String project, int limit, int offset) {
sql = sql.trim();
sql = sql.replace("\r", " ").replace("\n", System.getProperty("line.separator"));
@@ -65,7 +66,7 @@ public class QueryUtil {
initQueryTransformers();
}
for (IQueryTransformer t : queryTransformers) {
- sql = t.transform(sql);
+ sql = t.transform(sql, project);
}
return sql;
}
@@ -100,7 +101,7 @@ public class QueryUtil {
private static final Pattern PTN_HAVING_ESCAPE_FUNCTION = Pattern.compile("\\{fn" + "(.*?)" + "\\}", Pattern.CASE_INSENSITIVE);
@Override
- public String transform(String sql) {
+ public String transform(String sql, String project) {
Matcher m;
// Case fn{ EXTRACT(...) }
http://git-wip-us.apache.org/repos/asf/kylin/blob/05631b58/query/src/test/java/org/apache/kylin/query/util/CognosParenthesesEscapeTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/CognosParenthesesEscapeTest.java b/query/src/test/java/org/apache/kylin/query/util/CognosParenthesesEscapeTest.java
index 153c097..7825dbd 100644
--- a/query/src/test/java/org/apache/kylin/query/util/CognosParenthesesEscapeTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/CognosParenthesesEscapeTest.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.kylin.query.util;
import java.io.File;
@@ -33,16 +34,18 @@ public class CognosParenthesesEscapeTest {
CognosParenthesesEscape escape = new CognosParenthesesEscape();
String data = " from ((a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3) inner join c as cc on a.x1=cc.z1 ) join d dd on a.x1=d.w1 and a.x2 =d.w2 ";
String expected = " from a left outer join b on a.x1 = b.y1 and a.x2=b.y2 and a.x3= b.y3 inner join c as cc on a.x1=cc.z1 join d dd on a.x1=d.w1 and a.x2 =d.w2 ";
- String transformed = escape.transform(data);
+ String transformed = escape.transform(data, null);
Assert.assertEquals(expected, transformed);
}
@Test
public void advanced1Test() throws IOException {
CognosParenthesesEscape escape = new CognosParenthesesEscape();
- String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql"), Charset.defaultCharset());
- String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql.expected"), Charset.defaultCharset());
- String transformed = escape.transform(query);
+ String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql"),
+ Charset.defaultCharset());
+ String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query01.sql.expected"),
+ Charset.defaultCharset());
+ String transformed = escape.transform(query, null);
//System.out.println(transformed);
Assert.assertEquals(expected, transformed);
}
@@ -50,9 +53,11 @@ public class CognosParenthesesEscapeTest {
@Test
public void advanced2Test() throws IOException {
CognosParenthesesEscape escape = new CognosParenthesesEscape();
- String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query02.sql"), Charset.defaultCharset());
- String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query02.sql.expected"), Charset.defaultCharset());
- String transformed = escape.transform(query);
+ String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query02.sql"),
+ Charset.defaultCharset());
+ String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query02.sql.expected"),
+ Charset.defaultCharset());
+ String transformed = escape.transform(query, null);
//System.out.println(transformed);
Assert.assertEquals(expected, transformed);
}
@@ -60,9 +65,11 @@ public class CognosParenthesesEscapeTest {
@Test
public void advanced3Test() throws IOException {
CognosParenthesesEscape escape = new CognosParenthesesEscape();
- String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query03.sql"), Charset.defaultCharset());
- String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query03.sql.expected"), Charset.defaultCharset());
- String transformed = escape.transform(query);
+ String query = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query03.sql"),
+ Charset.defaultCharset());
+ String expected = FileUtils.readFileToString(new File("src/test/resources/query/cognos/query03.sql.expected"),
+ Charset.defaultCharset());
+ String transformed = escape.transform(query, null);
//System.out.println(transformed);
Assert.assertEquals(expected, transformed);
}
@@ -70,11 +77,12 @@ public class CognosParenthesesEscapeTest {
@Test
public void proguardTest() throws IOException {
CognosParenthesesEscape escape = new CognosParenthesesEscape();
- Collection<File> files = FileUtils.listFiles(new File("../kylin-it/src/test/resources"), new String[] { "sql" }, true);
+ Collection<File> files = FileUtils.listFiles(new File("../kylin-it/src/test/resources"), new String[] { "sql" },
+ true);
for (File f : files) {
System.out.println("checking " + f.getAbsolutePath());
String query = FileUtils.readFileToString(f, Charset.defaultCharset());
- String transformed = escape.transform(query);
+ String transformed = escape.transform(query, null);
Assert.assertEquals(query, transformed);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/05631b58/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java b/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
new file mode 100644
index 0000000..c3efe8d
--- /dev/null
+++ b/query/src/test/java/org/apache/kylin/query/util/ConvertToComputedColumnTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.util;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableSortedMap;
+
+public class ConvertToComputedColumnTest {
+ @Test
+ public void testEqual() throws SqlParseException {
+ String sql0 = "select a.a + a.b + a.c from t as a";
+ String sql1 = "select (((a . a + a.b + a.c))) from t as a";
+ String sql2 = "select (a + b) + c from t";
+ String sql3 = "select a.a + (a.b + a.c) from t as a";
+
+ SqlNode sn0 = getSelectNode(sql0);
+ SqlNode sn1 = getSelectNode(sql1);
+ SqlNode sn2 = getSelectNode(sql2);
+ SqlNode sn3 = getSelectNode(sql3);
+
+ Assert.assertEquals(true, ConvertToComputedColumn.isNodeEqual(sn0, sn1));
+ Assert.assertEquals(true, ConvertToComputedColumn.isNodeEqual(sn0, sn2));
+ Assert.assertEquals(false, ConvertToComputedColumn.isNodeEqual(sn0, sn3));
+ }
+
+ @Test
+ public void testErrorCase() {
+ //computed column is null or empty
+ String sql = "select a from t";
+ Map<String, String> map = new HashMap<>();
+ ImmutableSortedMap<String, String> computedColumns = ConvertToComputedColumn.getMapSortedByValue(map);
+ Assert.assertEquals("select a from t", ConvertToComputedColumn.replaceComputedColumn(sql, null));
+ Assert.assertEquals("select a from t", ConvertToComputedColumn.replaceComputedColumn(sql, computedColumns));
+
+ //input is null or empty or parse error
+ String sql1 = "";
+ String sql2 = "select sum(a from t";
+ Map<String, String> map2 = new HashMap<>();
+ map2.put("cc", "a + b");
+ ImmutableSortedMap<String, String> computedColumns2 = ConvertToComputedColumn.getMapSortedByValue(map2);
+ Assert.assertEquals("", ConvertToComputedColumn.replaceComputedColumn(null, computedColumns2));
+ Assert.assertEquals("", ConvertToComputedColumn.replaceComputedColumn(sql1, computedColumns2));
+ Assert.assertEquals("select sum(a from t",
+ ConvertToComputedColumn.replaceComputedColumn(sql2, computedColumns2));
+ }
+
+ @Test
+ public void testReplaceComputedColumn() throws SqlParseException {
+ String sql0 = "select (\"DB\".\"t1\" . \"a\" + DB.t1.b + DB.t1.c) as c, substring(substring(lstg_format_name,1,3),1,3) as d from table1 as t1 group by t1.a+ t1.b + t1.c having t1.a+t1.b+t1.c > 100 order by t1.a +t1.b +t1.c";
+ String sql1 = "select sum(sum(a)) from t";
+ String sql2 = "select t1.a + t1.b as aa, t2.c + t2.d as bb from table1 t1,table2 t2 where t1.a + t1.b > t2.c + t2.d order by t1.a + t1.b";
+ String sql3 = "select substring(substring(lstg_format_name,1,3),1,3) from a";
+
+ String expr0 = "a + b + c";
+ String expr1 = "sum(a)";
+ String expr2 = "a + b";
+ String expr3 = "c + d";
+ String expr = "substring(substring(lstg_format_name,1,3),1,3)";
+
+ Map<String, String> map = new HashMap<>();
+ map.put("cc0", expr0);
+ map.put("cc1", expr1);
+ map.put("cc2", expr2);
+ map.put("cc3", expr3);
+ map.put("cc", expr);
+
+ ImmutableSortedMap<String, String> computedColumns = ConvertToComputedColumn.getMapSortedByValue(map);
+ Assert.assertEquals(
+ "select (DB.t1.cc0) as c, cc as d from table1 as t1 group by T1.cc0 having T1.cc0 > 100 order by T1.cc0",
+ ConvertToComputedColumn.replaceComputedColumn(sql0, computedColumns));
+ Assert.assertEquals("select sum(cc1) from t",
+ ConvertToComputedColumn.replaceComputedColumn(sql1, computedColumns));
+ Assert.assertEquals(
+ "select T1.cc2 as aa, T2.cc3 as bb from table1 t1,table2 t2 where T1.cc2 > T2.cc3 order by T1.cc2",
+ ConvertToComputedColumn.replaceComputedColumn(sql2, computedColumns));
+ Assert.assertEquals("select cc from a", ConvertToComputedColumn.replaceComputedColumn(sql3, computedColumns));
+
+ }
+
+ private static SqlNode getSelectNode(String sql) throws SqlParseException {
+ return ((SqlSelect) ConvertToComputedColumn.parse(sql)).getSelectList().get(0);
+ }
+
+ @Test
+ public void testTwoCCHasSameSubExp() {
+ String sql0 = "select a + b + c from t order by a + b";
+
+ String expr0 = "a + b";
+ String expr1 = "a + b + c";
+
+ Map<String, String> map = new HashMap<>();
+ map.put("cc1", expr0);
+ map.put("cc0", expr1);
+ ImmutableSortedMap<String, String> computedColumns = ConvertToComputedColumn.getMapSortedByValue(map);
+ Assert.assertEquals("select cc0 from t order by cc1",
+ ConvertToComputedColumn.replaceComputedColumn(sql0, computedColumns));
+
+ //防止添加的顺序造成影响
+ String expr11 = "a + b + c";
+ String expr00 = "a + b";
+
+ Map<String, String> map2 = new HashMap<>();
+ map2.put("cc0", expr11);
+ map2.put("cc1", expr00);
+ ImmutableSortedMap<String, String> computedColumns1 = ConvertToComputedColumn.getMapSortedByValue(map2);
+ Assert.assertEquals("select cc0 from t order by cc1",
+ ConvertToComputedColumn.replaceComputedColumn(sql0, computedColumns1));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/05631b58/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
index a1edd89..f168d1e 100644
--- a/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
+++ b/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -40,12 +40,12 @@ public class QueryUtilTest extends LocalFileMetadataTestCase {
public void testMassageSql() {
{
String sql = "select ( date '2001-09-28' + interval floor(1.2) day) from test_kylin_fact";
- String s = QueryUtil.massageSql(sql, 0, 0);
+ String s = QueryUtil.massageSql(sql, null, 0, 0);
Assert.assertEquals("select ( date '2001-09-28' + interval '1' day) from test_kylin_fact", s);
}
{
String sql = "select ( date '2001-09-28' + interval floor(2) month) from test_kylin_fact group by ( date '2001-09-28' + interval floor(2) month)";
- String s = QueryUtil.massageSql(sql, 0, 0);
+ String s = QueryUtil.massageSql(sql, null, 0, 0);
Assert.assertEquals("select ( date '2001-09-28' + interval '2' month) from test_kylin_fact group by ( date '2001-09-28' + interval '2' month)", s);
}
}
@@ -54,7 +54,7 @@ public class QueryUtilTest extends LocalFileMetadataTestCase {
public void testKeywordDefaultDirtyHack() {
{
String sql = "select * from DEFAULT.TEST_KYLIN_FACT";
- String s = QueryUtil.massageSql(sql, 0, 0);
+ String s = QueryUtil.massageSql(sql, null, 0, 0);
Assert.assertEquals("select * from \"DEFAULT\".TEST_KYLIN_FACT", s);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/05631b58/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index d6554fc..8d18901 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -471,7 +471,7 @@ public class QueryService extends BasicService {
return fakeResponse;
}
- String correctedSql = QueryUtil.massageSql(sqlRequest.getSql(), sqlRequest.getLimit(), sqlRequest.getOffset());
+ String correctedSql = QueryUtil.massageSql(sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(), sqlRequest.getOffset());
if (!correctedSql.equals(sqlRequest.getSql())) {
logger.info("The corrected query: " + correctedSql);
[15/50] kylin git commit: KYLIN-2676 Add UUID into excludes list
Posted by li...@apache.org.
KYLIN-2676 Add UUID into excludes list
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/66dc5418
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/66dc5418
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/66dc5418
Branch: refs/heads/master
Commit: 66dc5418ce9860103a67398256ee11a7e428d4a0
Parents: ea06950
Author: auphyroc99 <45...@qq.com>
Authored: Fri Jun 16 15:32:33 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Mon Jun 19 13:23:47 2017 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/ResourceTool.java | 40 +++++++++++++++-----
1 file changed, 31 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/66dc5418/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 6ba68ae..6a73e12 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -37,6 +37,8 @@ public class ResourceTool {
private static String[] excludes = null;
private static final Logger logger = LoggerFactory.getLogger(ResourceTool.class);
+ private static final String[] IMMUTABLE_PREFIX = { "/UUID" };
+
public static void main(String[] args) throws IOException {
args = StringUtil.filterSystemArgs(args);
@@ -52,13 +54,15 @@ public class ResourceTool {
String include = System.getProperty("include");
if (include != null) {
- setIncludes(include.split("\\s*,\\s*"));
+ addIncludes(include.split("\\s*,\\s*"));
}
String exclude = System.getProperty("exclude");
if (exclude != null) {
- setExcludes(exclude.split("\\s*,\\s*"));
+ addExcludes(exclude.split("\\s*,\\s*"));
}
+ addExcludes(IMMUTABLE_PREFIX);
+
String cmd = args[0];
switch (cmd) {
case "reset":
@@ -91,16 +95,34 @@ public class ResourceTool {
return includes;
}
- public static void setIncludes(String[] arg) {
- includes = arg;
+ public static void addIncludes(String[] arg) {
+ if (arg != null) {
+ if (includes != null) {
+ String[] nIncludes = new String[includes.length + arg.length];
+ System.arraycopy(includes, 0, nIncludes, 0, includes.length);
+ System.arraycopy(arg, 0, nIncludes, includes.length, arg.length);
+ includes = nIncludes;
+ } else {
+ includes = arg;
+ }
+ }
}
public static String[] getExcludes() {
return excludes;
}
- public static void setExcludes(String[] arg) {
- excludes = arg;
+ public static void addExcludes(String[] arg) {
+ if (arg != null) {
+ if (excludes != null) {
+ String[] nExcludes = new String[excludes.length + arg.length];
+ System.arraycopy(excludes, 0, nExcludes, 0, excludes.length);
+ System.arraycopy(arg, 0, nExcludes, excludes.length, arg.length);
+ excludes = nExcludes;
+ } else {
+ excludes = arg;
+ }
+ }
}
public static String cat(KylinConfig config, String path) throws IOException {
@@ -134,16 +156,16 @@ public class ResourceTool {
ResourceStore dst = ResourceStore.getStore(dstConfig);
logger.info("Copy from {} to {}", src, dst);
-
+
copyR(src, dst, path);
}
public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths) throws IOException {
ResourceStore src = ResourceStore.getStore(srcConfig);
ResourceStore dst = ResourceStore.getStore(dstConfig);
-
+
logger.info("Copy from {} to {}", src, dst);
-
+
for (String path : paths) {
copyR(src, dst, path);
}
[24/50] kylin git commit: disable csrf for spring 4
Posted by li...@apache.org.
disable csrf for spring 4
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7661ad79
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7661ad79
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7661ad79
Branch: refs/heads/master
Commit: 7661ad79b40e9defcd02019f9b55ce0299459dd2
Parents: 73faf99
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Jun 20 18:40:38 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Tue Jun 20 19:14:02 2017 +0800
----------------------------------------------------------------------
server/src/main/resources/kylinSecurity.xml | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7661ad79/server/src/main/resources/kylinSecurity.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/kylinSecurity.xml b/server/src/main/resources/kylinSecurity.xml
index 43a0082..506b2f1 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -29,6 +29,8 @@
<scr:expression-handler ref="expressionHandler"/>
</scr:global-method-security>
+ <scr:csrf disabled="true"/>
+
<!-- acl config -->
<bean id="aclPermissionFactory" class="org.apache.kylin.rest.security.AclPermissionFactory"/>
[06/50] kylin git commit: #1173 refine Cube and Model search API
Posted by li...@apache.org.
#1173 refine Cube and Model search API
provide both exact-match and fuzzy search API for Cube and Model
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f32165ef
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f32165ef
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f32165ef
Branch: refs/heads/master
Commit: f32165ef3c89ed6a6004a63d28f7227e6cd6d1e6
Parents: 56e9fb9
Author: Roger Shi <ro...@hotmail.com>
Authored: Thu Jun 15 20:56:07 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 15 21:16:29 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/controller/CubeController.java | 2 +-
.../kylin/rest/controller2/CubeControllerV2.java | 3 ++-
.../kylin/rest/controller2/ModelControllerV2.java | 3 ++-
.../org/apache/kylin/rest/service/CubeService.java | 5 +++--
.../org/apache/kylin/rest/service/ModelService.java | 15 ++++++++-------
.../org/apache/kylin/rest/service/QueryService.java | 2 +-
.../apache/kylin/rest/service/CubeServiceTest.java | 2 +-
.../apache/kylin/rest/service/ModelServiceTest.java | 6 +++---
8 files changed, 21 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f32165ef/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index bfa5603..14c80a0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -101,7 +101,7 @@ public class CubeController extends BasicController {
@ResponseBody
public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
List<CubeInstance> cubes;
- cubes = cubeService.listAllCubes(cubeName, projectName, modelName);
+ cubes = cubeService.listAllCubes(cubeName, projectName, modelName, true);
int climit = (null == limit) ? cubes.size() : limit;
int coffset = (null == offset) ? 0 : offset;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f32165ef/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
index 309fffc..720cf76 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/CubeControllerV2.java
@@ -102,6 +102,7 @@ public class CubeControllerV2 extends BasicController {
@RequestMapping(value = "", method = { RequestMethod.GET }, produces = { "application/vnd.apache.kylin-v2+json" })
@ResponseBody
public EnvelopeResponse getCubesPaging(@RequestParam(value = "cubeName", required = false) String cubeName,
+ @RequestParam(value = "exactMatch", required = false, defaultValue = "true") boolean exactMatch,
@RequestParam(value = "modelName", required = false) String modelName,
@RequestParam(value = "projectName", required = false) String projectName,
@RequestParam(value = "pageOffset", required = false, defaultValue = "0") Integer pageOffset,
@@ -109,7 +110,7 @@ public class CubeControllerV2 extends BasicController {
HashMap<String, Object> data = new HashMap<String, Object>();
List<CubeInstanceResponse> response = new ArrayList<CubeInstanceResponse>();
- List<CubeInstance> cubes = cubeService.listAllCubes(cubeName, projectName, modelName);
+ List<CubeInstance> cubes = cubeService.listAllCubes(cubeName, projectName, modelName, exactMatch);
// official cubes
for (CubeInstance cube : cubes) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f32165ef/server-base/src/main/java/org/apache/kylin/rest/controller2/ModelControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/ModelControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/ModelControllerV2.java
index e775c9c..58f6bee 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/ModelControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/ModelControllerV2.java
@@ -92,6 +92,7 @@ public class ModelControllerV2 extends BasicController {
@RequestMapping(value = "", method = { RequestMethod.GET }, produces = { "application/vnd.apache.kylin-v2+json" })
@ResponseBody
public EnvelopeResponse getModelsPaging(@RequestParam(value = "modelName", required = false) String modelName,
+ @RequestParam(value = "exactMatch", required = false, defaultValue = "true") boolean exactMatch,
@RequestParam(value = "projectName", required = false) String projectName,
@RequestParam(value = "pageOffset", required = false, defaultValue = "0") Integer pageOffset,
@RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize)
@@ -100,7 +101,7 @@ public class ModelControllerV2 extends BasicController {
List<DataModelDescResponse> response = new ArrayList<DataModelDescResponse>();
// official models
- for (DataModelDesc m : modelService.listAllModels(modelName, projectName)) {
+ for (DataModelDesc m : modelService.listAllModels(modelName, projectName, exactMatch)) {
Preconditions.checkState(!m.isDraft());
DataModelDescResponse r = new DataModelDescResponse(m);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f32165ef/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 4a03e05..74d8578 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -97,7 +97,7 @@ public class CubeService extends BasicService {
private ModelService modelService;
@PostFilter(Constant.ACCESS_POST_FILTER_READ)
- public List<CubeInstance> listAllCubes(final String cubeName, final String projectName, final String modelName) {
+ public List<CubeInstance> listAllCubes(final String cubeName, final String projectName, final String modelName, boolean exactMatch) {
List<CubeInstance> cubeInstances = null;
ProjectInstance project = (null != projectName) ? getProjectManager().getProject(projectName) : null;
@@ -124,7 +124,8 @@ public class CubeService extends BasicService {
List<CubeInstance> filterCubes = new ArrayList<CubeInstance>();
for (CubeInstance cubeInstance : filterModelCubes) {
boolean isCubeMatch = (null == cubeName)
- || cubeInstance.getName().toLowerCase().contains(cubeName.toLowerCase());
+ || (!exactMatch && cubeInstance.getName().toLowerCase().contains(cubeName.toLowerCase())) ||
+ (exactMatch && cubeInstance.getName().toLowerCase().equals(cubeName.toLowerCase()));
if (isCubeMatch) {
filterCubes.add(cubeInstance);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f32165ef/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 14ee65a..fa54eaa 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -73,7 +73,7 @@ public class ModelService extends BasicService {
private CubeService cubeService;
@PostFilter(Constant.ACCESS_POST_FILTER_READ)
- public List<DataModelDesc> listAllModels(final String modelName, final String projectName) throws IOException {
+ public List<DataModelDesc> listAllModels(final String modelName, final String projectName, boolean exactMatch) throws IOException {
List<DataModelDesc> models;
ProjectInstance project = (null != projectName) ? getProjectManager().getProject(projectName) : null;
@@ -86,7 +86,8 @@ public class ModelService extends BasicService {
List<DataModelDesc> filterModels = new ArrayList<DataModelDesc>();
for (DataModelDesc modelDesc : models) {
boolean isModelMatch = (null == modelName) || modelName.length() == 0
- || modelDesc.getName().toLowerCase().equals(modelName.toLowerCase());
+ || (exactMatch && modelDesc.getName().toLowerCase().equals(modelName.toLowerCase()))
+ || (!exactMatch && modelDesc.getName().toLowerCase().contains(modelName.toLowerCase()));
if (isModelMatch) {
filterModels.add(modelDesc);
@@ -99,7 +100,7 @@ public class ModelService extends BasicService {
public List<DataModelDesc> getModels(final String modelName, final String projectName, final Integer limit,
final Integer offset) throws IOException {
- List<DataModelDesc> modelDescs = listAllModels(modelName, projectName);
+ List<DataModelDesc> modelDescs = listAllModels(modelName, projectName, true);
if (limit == null || offset == null) {
return modelDescs;
@@ -182,7 +183,7 @@ public class ModelService extends BasicService {
public Map<TblColRef, Set<CubeInstance>> getUsedDimCols(String modelName) {
Map<TblColRef, Set<CubeInstance>> ret = Maps.newHashMap();
- List<CubeInstance> cubeInstances = cubeService.listAllCubes(null, null, modelName);
+ List<CubeInstance> cubeInstances = cubeService.listAllCubes(null, null, modelName, true);
for (CubeInstance cubeInstance : cubeInstances) {
CubeDesc cubeDesc = cubeInstance.getDescriptor();
for (TblColRef tblColRef : cubeDesc.listDimensionColumnsIncludingDerived()) {
@@ -199,7 +200,7 @@ public class ModelService extends BasicService {
public Map<TblColRef, Set<CubeInstance>> getUsedNonDimCols(String modelName) {
Map<TblColRef, Set<CubeInstance>> ret = Maps.newHashMap();
- List<CubeInstance> cubeInstances = cubeService.listAllCubes(null, null, modelName);
+ List<CubeInstance> cubeInstances = cubeService.listAllCubes(null, null, modelName, true);
for (CubeInstance cubeInstance : cubeInstances) {
CubeDesc cubeDesc = cubeInstance.getDescriptor();
Set<TblColRef> tblColRefs = Sets.newHashSet(cubeDesc.listAllColumns());//make a copy
@@ -218,7 +219,7 @@ public class ModelService extends BasicService {
private boolean validateUpdatingModel(DataModelDesc dataModelDesc) throws IOException {
String modelName = dataModelDesc.getName();
- List<CubeInstance> cubes = cubeService.listAllCubes(null, null, modelName);
+ List<CubeInstance> cubes = cubeService.listAllCubes(null, null, modelName, true);
if (cubes != null && cubes.size() != 0) {
dataModelDesc.init(getConfig(), getMetadataManager().getAllTablesMap(),
getMetadataManager().getCcInfoMap());
@@ -255,7 +256,7 @@ public class ModelService extends BasicService {
return false;
}
- DataModelDesc originDataModelDesc = listAllModels(modelName, null).get(0);
+ DataModelDesc originDataModelDesc = listAllModels(modelName, null, true).get(0);
if (!dataModelDesc.getRootFactTable().equals(originDataModelDesc.getRootFactTable()))
return false;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f32165ef/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 1b118a3..bf32140 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -610,7 +610,7 @@ public class QueryService extends BasicService {
ProjectInstance projectInstance = getProjectManager().getProject(project);
for (String modelName : projectInstance.getModels()) {
- DataModelDesc dataModelDesc = modelService.listAllModels(modelName, project).get(0);
+ DataModelDesc dataModelDesc = modelService.listAllModels(modelName, project, true).get(0);
if (!dataModelDesc.isDraft()) {
// update table type: FACT
http://git-wip-us.apache.org/repos/asf/kylin/blob/f32165ef/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
index a190d6d..96146ef 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
@@ -51,7 +51,7 @@ public class CubeServiceTest extends ServiceTestBase {
Assert.assertNotNull(cubeService.getMetadataManager());
Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
- List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null);
+ List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null, true);
Assert.assertNotNull(cubes);
CubeInstance cube = cubes.get(0);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f32165ef/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 2012a05..b78b18e 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -48,7 +48,7 @@ public class ModelServiceTest extends ServiceTestBase {
@Test
public void testSuccessModelUpdate() throws IOException, JobException {
- List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_inner_join_model", "default");
+ List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_inner_join_model", "default", true);
Assert.assertTrue(dataModelDescs.size() == 1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -64,7 +64,7 @@ public class ModelServiceTest extends ServiceTestBase {
@Test
public void testSuccessModelUpdateOnComputedColumn() throws IOException, JobException, NoSuchFieldException, IllegalAccessException {
- List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default");
+ List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default", true);
Assert.assertTrue(dataModelDescs.size() == 1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -83,7 +83,7 @@ public class ModelServiceTest extends ServiceTestBase {
expectedEx.expect(IllegalStateException.class);
expectedEx.expectMessage("Computed column named DEFAULT.TEST_KYLIN_FACT.DEAL_AMOUNT is already defined in other models: [DataModelDesc [name=ci_left_join_model], DataModelDesc [name=ci_inner_join_model]]. Please change another name, or try to keep consistent definition");
- List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default");
+ List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default", true);
Assert.assertTrue(dataModelDescs.size() == 1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
[21/50] kylin git commit: upgrade to Spring 4
Posted by li...@apache.org.
upgrade to Spring 4
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/247f3cd2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/247f3cd2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/247f3cd2
Branch: refs/heads/master
Commit: 247f3cd266c77480ac0f8d329e8e7adb617b5de0
Parents: 5a4e465
Author: Yang Li <li...@apache.org>
Authored: Mon Jun 19 21:58:25 2017 +0800
Committer: Roger Shi <ro...@gmail.com>
Committed: Tue Jun 20 15:28:30 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 3 ++
pom.xml | 28 ++++++++++++++----
.../rest/controller2/UserControllerV2.java | 7 +++--
.../kylin/rest/response/EnvelopeResponse.java | 6 ++--
.../apache/kylin/rest/service/UserService.java | 6 ++--
.../src/main/resources/applicationContext.xml | 31 +++++++-------------
server/src/main/resources/kylinSecurity.xml | 17 ++++++-----
7 files changed, 53 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/247f3cd2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e64dddb..2ccf3cf 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -62,6 +62,9 @@ abstract public class KylinConfigBase implements Serializable {
public static String getKylinHome() {
String kylinHome = System.getenv("KYLIN_HOME");
if (StringUtils.isEmpty(kylinHome)) {
+ kylinHome = System.getProperty("KYLIN_HOME");
+ }
+ if (StringUtils.isEmpty(kylinHome)) {
logger.warn("KYLIN_HOME was not set");
}
return kylinHome;
http://git-wip-us.apache.org/repos/asf/kylin/blob/247f3cd2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f887c8d..506d734 100644
--- a/pom.xml
+++ b/pom.xml
@@ -114,11 +114,11 @@
<tomcat.version>7.0.69</tomcat.version>
<t-digest.version>3.1</t-digest.version>
- <!-- REST Service -->
- <spring.framework.version>3.2.17.RELEASE</spring.framework.version>
- <spring.framework.security.version>3.1.2.RELEASE</spring.framework.security.version>
- <spring.framework.security.extensions.version>1.0.2.RELEASE
- </spring.framework.security.extensions.version>
+ <!-- REST Service, ref https://github.com/spring-projects/spring-boot/blob/v1.3.8.RELEASE/spring-boot-dependencies/pom.xml -->
+ <spring.boot.version>1.3.8.RELEASE</spring.boot.version>
+ <spring.framework.version>4.2.8.RELEASE</spring.framework.version>
+ <spring.framework.security.version>4.0.4.RELEASE</spring.framework.security.version>
+ <spring.framework.security.extensions.version>1.0.2.RELEASE</spring.framework.security.extensions.version>
<opensaml.version>2.6.4</opensaml.version>
<aspectj.version>1.8.9</aspectj.version>
@@ -768,14 +768,30 @@
</dependency>
<dependency>
<groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-config</artifactId>
+ <version>${spring.framework.security.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-core</artifactId>
+ <version>${spring.framework.security.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
<artifactId>spring-security-ldap</artifactId>
<version>${spring.framework.security.version}</version>
</dependency>
<dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-web</artifactId>
+ <version>${spring.framework.security.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.springframework.security.extensions</groupId>
<artifactId>spring-security-saml2-core</artifactId>
<version>${spring.framework.security.extensions.version}</version>
</dependency>
+
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
@@ -1046,7 +1062,7 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
- <version>1.3.6.RELEASE</version>
+ <version>${spring.boot.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/kylin/blob/247f3cd2/server-base/src/main/java/org/apache/kylin/rest/controller2/UserControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/UserControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/UserControllerV2.java
index 2f7a404..c0f5c17 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/UserControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/UserControllerV2.java
@@ -19,6 +19,7 @@
package org.apache.kylin.rest.controller2;
import java.io.IOException;
+import java.util.List;
import org.apache.kylin.rest.controller.BasicController;
import org.apache.kylin.rest.exception.UnauthorizedException;
@@ -59,7 +60,7 @@ public class UserControllerV2 extends BasicController {
@RequestMapping(value = "/authentication", method = RequestMethod.POST, produces = {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
- public EnvelopeResponse authenticateV2() {
+ public EnvelopeResponse<UserDetails> authenticateV2() {
EnvelopeResponse response = authenticatedUserV2();
logger.debug("User login: {}", response.data);
return response;
@@ -68,7 +69,7 @@ public class UserControllerV2 extends BasicController {
@RequestMapping(value = "/authentication", method = RequestMethod.GET, produces = {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
- public EnvelopeResponse authenticatedUserV2() {
+ public EnvelopeResponse<UserDetails> authenticatedUserV2() {
Message msg = MsgPicker.getMsg();
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
@@ -95,7 +96,7 @@ public class UserControllerV2 extends BasicController {
@RequestMapping(value = "/authentication/authorities", method = RequestMethod.GET, produces = {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
- public EnvelopeResponse getAuthoritiesV2() throws IOException {
+ public EnvelopeResponse<List<String>> getAuthoritiesV2() throws IOException {
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, userService.listUserAuthorities(), "");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/247f3cd2/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
index 7855dee..8350536 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/EnvelopeResponse.java
@@ -18,17 +18,17 @@
package org.apache.kylin.rest.response;
-public class EnvelopeResponse {
+public class EnvelopeResponse<T> {
public String code;
- public Object data;
+ public T data;
public String msg;
//only for child
protected EnvelopeResponse() {
}
- public EnvelopeResponse(String code, Object data, String msg) {
+ public EnvelopeResponse(String code, T data, String msg) {
this.code = code;
this.data = data;
this.msg = msg;
http://git-wip-us.apache.org/repos/asf/kylin/blob/247f3cd2/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
index 5452543..9adfcb8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/UserService.java
@@ -30,13 +30,11 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.GrantedAuthority;
import org.springframework.security.core.userdetails.User;
import org.springframework.security.core.userdetails.UserDetails;
@@ -61,13 +59,13 @@ public class UserService implements UserDetailsManager {
}
@Override
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+ //@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) --- DON'T DO THIS, CAUSES CIRCULAR DEPENDENCY BETWEEN UserService & AclService
public void createUser(UserDetails user) {
updateUser(user);
}
@Override
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+ //@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) --- DON'T DO THIS, CAUSES CIRCULAR DEPENDENCY BETWEEN UserService & AclService
public void updateUser(UserDetails user) {
try {
deleteUser(user.getUsername());
http://git-wip-us.apache.org/repos/asf/kylin/blob/247f3cd2/server/src/main/resources/applicationContext.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/applicationContext.xml b/server/src/main/resources/applicationContext.xml
index 8416f25..88286b5 100644
--- a/server/src/main/resources/applicationContext.xml
+++ b/server/src/main/resources/applicationContext.xml
@@ -21,15 +21,15 @@
xmlns:cache="http://www.springframework.org/schema/cache"
xmlns:p="http://www.springframework.org/schema/p"
xsi:schemaLocation="http://www.springframework.org/schema/beans
- http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+ http://www.springframework.org/schema/beans/spring-beans-4.2.xsd
http://www.springframework.org/schema/context
- http://www.springframework.org/schema/context/spring-context-3.1.xsd
+ http://www.springframework.org/schema/context/spring-context-4.2.xsd
http://www.springframework.org/schema/task
- http://www.springframework.org/schema/task/spring-task-3.1.xsd
+ http://www.springframework.org/schema/task/spring-task-4.2.xsd
http://www.springframework.org/schema/mvc
- http://www.springframework.org/schema/mvc/spring-mvc-3.1.xsd
+ http://www.springframework.org/schema/mvc/spring-mvc-4.2.xsd
http://www.springframework.org/schema/aop
- http://www.springframework.org/schema/aop/spring-aop-3.1.xsd
+ http://www.springframework.org/schema/aop/spring-aop-4.2.xsd
http://www.springframework.org/schema/cache
http://www.springframework.org/schema/cache/spring-cache.xsd">
@@ -48,8 +48,7 @@
<!-- Rest service binding -->
- <bean
- class="org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping"/>
+ <bean class="org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerMapping"/>
<bean id="mappingJacksonHttpMessageConverter"
class="org.springframework.http.converter.json.MappingJackson2HttpMessageConverter"/>
@@ -58,8 +57,7 @@
<bean id="formHttpMessageConverter"
class="org.springframework.http.converter.FormHttpMessageConverter"/>
- <bean
- class="org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter">
+ <bean class="org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter">
<property name="messageConverters">
<list>
<ref bean="mappingJacksonHttpMessageConverter"/>
@@ -69,19 +67,11 @@
</property>
</bean>
- <bean
- class="org.springframework.web.servlet.view.ContentNegotiatingViewResolver">
- <property name="mediaTypes">
- <map>
- <entry key="html" value="text/html"/>
- <entry key="json" value="application/json"/>
- </map>
- </property>
+ <bean class="org.springframework.web.servlet.view.ContentNegotiatingViewResolver">
<property name="viewResolvers">
<list>
<bean class="org.springframework.web.servlet.view.BeanNameViewResolver"/>
- <bean
- class="org.springframework.web.servlet.view.InternalResourceViewResolver">
+ <bean class="org.springframework.web.servlet.view.InternalResourceViewResolver">
<!-- <property name="prefix" value="/WEB-INF/jsp/"/> -->
<property name="suffix" value=".jsp"/>
</bean>
@@ -89,8 +79,7 @@
</property>
<property name="defaultViews">
<list>
- <bean
- class="org.springframework.web.servlet.view.json.MappingJackson2JsonView"/>
+ <bean class="org.springframework.web.servlet.view.json.MappingJackson2JsonView"/>
</list>
</property>
</bean>
http://git-wip-us.apache.org/repos/asf/kylin/blob/247f3cd2/server/src/main/resources/kylinSecurity.xml
----------------------------------------------------------------------
diff --git a/server/src/main/resources/kylinSecurity.xml b/server/src/main/resources/kylinSecurity.xml
index 374341c..43a0082 100644
--- a/server/src/main/resources/kylinSecurity.xml
+++ b/server/src/main/resources/kylinSecurity.xml
@@ -17,12 +17,13 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:util="http://www.springframework.org/schema/util" xsi:schemaLocation="http://www.springframework.org/schema/beans
- http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+ http://www.springframework.org/schema/beans/spring-beans-4.2.xsd
http://www.springframework.org/schema/security
- http://www.springframework.org/schema/security/spring-security-3.1.xsd
- http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.1.xsd
-
- http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">
+ http://www.springframework.org/schema/security/spring-security-4.0.xsd
+ http://www.springframework.org/schema/util
+ http://www.springframework.org/schema/util/spring-util-4.2.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context.xsd">
<scr:global-method-security pre-post-annotations="enabled">
<scr:expression-handler ref="expressionHandler"/>
@@ -45,13 +46,13 @@
class="org.springframework.security.acls.domain.AclAuthorizationStrategyImpl">
<constructor-arg>
<list>
- <bean class="org.springframework.security.core.authority.GrantedAuthorityImpl">
+ <bean class="org.springframework.security.core.authority.SimpleGrantedAuthority">
<constructor-arg value="ROLE_ADMIN"/>
</bean>
- <bean class="org.springframework.security.core.authority.GrantedAuthorityImpl">
+ <bean class="org.springframework.security.core.authority.SimpleGrantedAuthority">
<constructor-arg value="ROLE_ADMIN"/>
</bean>
- <bean class="org.springframework.security.core.authority.GrantedAuthorityImpl">
+ <bean class="org.springframework.security.core.authority.SimpleGrantedAuthority">
<constructor-arg value="ROLE_ADMIN"/>
</bean>
</list>
[33/50] kylin git commit: minor, push down operator ||
Posted by li...@apache.org.
minor, push down operator ||
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0eeceeff
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0eeceeff
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0eeceeff
Branch: refs/heads/master
Commit: 0eeceeff43ff64bcd4baa1bc214e1288d7716f9e
Parents: 401bb0a
Author: Cheng Wang <ch...@kyligence.io>
Authored: Thu Jun 22 20:30:09 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 22 20:56:24 2017 +0800
----------------------------------------------------------------------
.../filter/BuiltInFunctionTupleFilter.java | 21 +++++++++++++++-----
.../metadata/filter/function/BuiltInMethod.java | 11 +++++++++-
2 files changed, 26 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0eeceeff/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
index 767d868..aa9cd3d 100755
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/BuiltInFunctionTupleFilter.java
@@ -24,6 +24,7 @@ import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.metadata.filter.function.BuiltInMethod;
@@ -33,6 +34,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.primitives.Primitives;
public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
@@ -49,12 +51,18 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
protected boolean isValidFunc = false;
protected boolean isReversed = false;
+ final static Map<String, String> converters = Maps.newHashMap();
+ static {
+ converters.put("||", "CONCAT");
+ }
+
public BuiltInFunctionTupleFilter(String name) {
this(name, null);
}
public BuiltInFunctionTupleFilter(String name, FilterOperatorEnum filterOperatorEnum) {
- super(Lists.<TupleFilter> newArrayList(), filterOperatorEnum == null ? FilterOperatorEnum.FUNCTION : filterOperatorEnum);
+ super(Lists.<TupleFilter> newArrayList(),
+ filterOperatorEnum == null ? FilterOperatorEnum.FUNCTION : filterOperatorEnum);
this.methodParams = Lists.newArrayList();
if (name != null) {
@@ -91,7 +99,8 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
if (columnContainerFilter instanceof ColumnTupleFilter)
methodParams.set(colPosition, (Serializable) input);
else if (columnContainerFilter instanceof BuiltInFunctionTupleFilter)
- methodParams.set(colPosition, (Serializable) ((BuiltInFunctionTupleFilter) columnContainerFilter).invokeFunction(input));
+ methodParams.set(colPosition,
+ (Serializable) ((BuiltInFunctionTupleFilter) columnContainerFilter).invokeFunction(input));
return method.invoke(null, (Object[]) (methodParams.toArray()));
}
@@ -128,7 +137,8 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
if (!Primitives.isWrapperType(clazz))
methodParams.add(constVal);
else
- methodParams.add((Serializable) clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, constVal)));
+ methodParams.add((Serializable) clazz
+ .cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null, constVal)));
} catch (Exception e) {
logger.warn("Reflection failed for methodParams. ", e);
isValidFunc = false;
@@ -186,8 +196,9 @@ public class BuiltInFunctionTupleFilter extends FunctionTupleFilter {
}
protected void initMethod() {
- if (BuiltInMethod.MAP.containsKey(name)) {
- this.method = BuiltInMethod.MAP.get(name).method;
+ String operator = BuiltInMethod.MAP.containsKey(name) ? name : converters.get(name);
+ if (operator != null) {
+ this.method = BuiltInMethod.MAP.get(operator).method;
isValidFunc = true;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0eeceeff/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
index 31ee297..e156174 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/function/BuiltInMethod.java
@@ -29,7 +29,7 @@ import org.apache.commons.lang3.reflect.MethodUtils;
import com.google.common.collect.ImmutableMap;
public enum BuiltInMethod {
- UPPER(BuiltInMethod.class, "upper", String.class), LOWER(BuiltInMethod.class, "lower", String.class), SUBSTRING(BuiltInMethod.class, "substring", String.class, int.class, int.class), CHAR_LENGTH(BuiltInMethod.class, "charLength", String.class), LIKE(BuiltInMethod.class, "like", String.class, String.class), INITCAP(BuiltInMethod.class, "initcap", String.class);
+ UPPER(BuiltInMethod.class, "upper", String.class), LOWER(BuiltInMethod.class, "lower", String.class), SUBSTRING(BuiltInMethod.class, "substring", String.class, int.class, int.class), CHAR_LENGTH(BuiltInMethod.class, "charLength", String.class), LIKE(BuiltInMethod.class, "like", String.class, String.class), INITCAP(BuiltInMethod.class, "initcap", String.class), CONCAT(BuiltInMethod.class, "concat", String.class, String.class);
public final Method method;
public static final ImmutableMap<String, BuiltInMethod> MAP;
@@ -140,4 +140,13 @@ public enum BuiltInMethod {
return s.toLowerCase();
}
+ /** SQL left || right */
+ public static String concat(String left, String right) {
+ if (left == null)
+ return right;
+ if (right == null)
+ return left;
+ return left.concat(right);
+ }
+
}
[13/50] kylin git commit: #1186 #1150 forbid project rename and
remove if it's not empty
Posted by li...@apache.org.
#1186 #1150 forbid project rename and remove if it's not empty
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0ab915da
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0ab915da
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0ab915da
Branch: refs/heads/master
Commit: 0ab915dad029440bf8b9c6091958f99d7cadd241
Parents: 70ad90c
Author: Roger Shi <ro...@hotmail.com>
Authored: Sat Jun 17 15:43:36 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Sat Jun 17 15:47:29 2017 +0800
----------------------------------------------------------------------
.../rest/controller2/ProjectControllerV2.java | 35 ++++++++++++++++----
.../org/apache/kylin/rest/msg/CnMessage.java | 7 ++++
.../java/org/apache/kylin/rest/msg/Message.java | 8 +++++
3 files changed, 44 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0ab915da/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
index 52e17b5..5a34807 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/ProjectControllerV2.java
@@ -32,6 +32,8 @@ import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.ProjectRequest;
import org.apache.kylin.rest.response.EnvelopeResponse;
import org.apache.kylin.rest.response.ResponseCode;
+import org.apache.kylin.rest.service.CubeService;
+import org.apache.kylin.rest.service.ModelService;
import org.apache.kylin.rest.service.ProjectService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,6 +62,14 @@ public class ProjectControllerV2 extends BasicController {
@Qualifier("projectService")
private ProjectService projectService;
+ @Autowired
+ @Qualifier("cubeMgmtService")
+ private CubeService cubeService;
+
+ @Autowired
+ @Qualifier("modelMgmtService")
+ private ModelService modelService;
+
@RequestMapping(value = "", method = { RequestMethod.GET }, produces = {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
@@ -123,21 +133,23 @@ public class ProjectControllerV2 extends BasicController {
ProjectInstance projectDesc = deserializeProjectDescV2(projectRequest);
- ProjectInstance updatedProj = null;
-
ProjectInstance currentProject = projectService.getProjectManager().getProject(formerProjectName);
if (currentProject == null) {
throw new BadRequestException(String.format(msg.getPROJECT_NOT_FOUND(), formerProjectName));
}
- updatedProj = projectService.updateProject(projectDesc, currentProject);
+ // cannot modify project name if it's not empty
+ if (!currentProject.getName().equals(projectDesc.getName()) && !isProjectEmpty(currentProject.getName())) {
+ throw new BadRequestException(msg.getRENAME_PROJECT_NOT_EMPTY());
+ }
+
+ ProjectInstance updatedProj = projectService.updateProject(projectDesc, currentProject);
return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, updatedProj, "");
}
private ProjectInstance deserializeProjectDescV2(ProjectRequest projectRequest) throws IOException {
- ProjectInstance projectDesc = null;
logger.debug("Saving project " + projectRequest.getProjectDescData());
- projectDesc = JsonUtil.readValue(projectRequest.getProjectDescData(), ProjectInstance.class);
+ ProjectInstance projectDesc = JsonUtil.readValue(projectRequest.getProjectDescData(), ProjectInstance.class);
return projectDesc;
}
@@ -145,9 +157,20 @@ public class ProjectControllerV2 extends BasicController {
"application/vnd.apache.kylin-v2+json" })
@ResponseBody
public void deleteProjectV2(@PathVariable String projectName) throws IOException {
-
+ Message msg = MsgPicker.getMsg();
ProjectInstance project = projectService.getProjectManager().getProject(projectName);
+ if (!isProjectEmpty(projectName)) {
+ throw new BadRequestException(msg.getDELETE_PROJECT_NOT_EMPTY());
+ }
+
projectService.deleteProject(projectName, project);
}
+ private boolean isProjectEmpty(String projectName) throws IOException {
+ return cubeService.listAllCubes(projectName).isEmpty()
+ && cubeService.listCubeDrafts(null, null, projectName).isEmpty()
+ && modelService.listAllModels(null, projectName, false).isEmpty()
+ && modelService.listModelDrafts(null, projectName).isEmpty();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0ab915da/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
index 1c6fb69..a828aa0 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/CnMessage.java
@@ -245,6 +245,13 @@ public class CnMessage extends Message {
return "找不到项目 '%s'";
}
+ public String getDELETE_PROJECT_NOT_EMPTY() {
+ return "不能删除该项目，如需要删除请先清空其中的Cube和Model";
+ }
+
+ public String getRENAME_PROJECT_NOT_EMPTY() {
+ return "不能重命名该项目，如果要重命名请先清空其中的Cube和Model";
+ }
// Table
public String getHIVE_TABLE_NOT_FOUND() {
return "找不到 Hive 表 '%s'";
http://git-wip-us.apache.org/repos/asf/kylin/blob/0ab915da/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
index f4bcda7..9ed38bb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
@@ -245,6 +245,14 @@ public class Message {
return "Cannot find project '%s'.";
}
+ public String getDELETE_PROJECT_NOT_EMPTY() {
+ return "Cannot delete non-empty project";
+ }
+
+ public String getRENAME_PROJECT_NOT_EMPTY() {
+ return "Cannot rename non-empty project";
+ }
+
// Table
public String getHIVE_TABLE_NOT_FOUND() {
return "Cannot find Hive table '%s'.";
[32/50] kylin git commit: minor, add test case for ||
Posted by li...@apache.org.
minor, add test case for ||
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3f9d158a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3f9d158a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3f9d158a
Branch: refs/heads/master
Commit: 3f9d158a13186306b666898ea607768094805559
Parents: 0eeceef
Author: Cheng Wang <ch...@kyligence.io>
Authored: Thu Jun 22 20:55:19 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 22 20:56:24 2017 +0800
----------------------------------------------------------------------
kylin-it/src/test/resources/query/sql/query96.sql | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3f9d158a/kylin-it/src/test/resources/query/sql/query96.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql/query96.sql b/kylin-it/src/test/resources/query/sql/query96.sql
index 4e80d59..eae99bc 100644
--- a/kylin-it/src/test/resources/query/sql/query96.sql
+++ b/kylin-it/src/test/resources/query/sql/query96.sql
@@ -18,5 +18,5 @@
select upper(lstg_format_name) as lstg_format_name, count(*) as cnt from test_kylin_fact
where lower(lstg_format_name)='abin' and substring(lstg_format_name,1,3) in ('ABI') and upper(lstg_format_name) > 'AAAA' and
-lower(lstg_format_name) like '%b%' and char_length(lstg_format_name) < 10 and char_length(lstg_format_name) > 3 and lstg_format_name||'a'='ABINa'
+lower(lstg_format_name) like '%b%' and char_length(lstg_format_name) < 10 and char_length(lstg_format_name) > 3 and 'abc'||lstg_format_name||'a'||'b'||'c'='abcABINabc'
group by lstg_format_name
\ No newline at end of file
[08/50] kylin git commit: KYLIN-2529 fix implementation switch for
threadLocal KylinConfig
Posted by li...@apache.org.
KYLIN-2529 fix implementation switch for threadLocal KylinConfig
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d659bade
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d659bade
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d659bade
Branch: refs/heads/master
Commit: d659bade9283ecbf99e31285e620e880ef6a06b1
Parents: 51f22e2
Author: lidongsjtu <li...@apache.org>
Authored: Thu Jun 15 18:31:02 2017 +0800
Committer: liyang-gmt8 <li...@apache.org>
Committed: Fri Jun 16 10:11:21 2017 +0800
----------------------------------------------------------------------
.../org/apache/kylin/engine/EngineFactory.java | 21 ++---
.../org/apache/kylin/job/SchedulerFactory.java | 23 ++---
.../org/apache/kylin/source/SourceFactory.java | 22 +++--
.../apache/kylin/storage/StorageFactory.java | 16 ++--
.../kylin/storage/StorageFactoryTest.java | 93 ++++++++++++++++++++
5 files changed, 133 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d659bade/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
index acaa7da..78b1efe 100644
--- a/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
+++ b/core-job/src/main/java/org/apache/kylin/engine/EngineFactory.java
@@ -18,8 +18,6 @@
package org.apache.kylin.engine;
-import java.util.Map;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ImplementationSwitch;
import org.apache.kylin.cube.CubeSegment;
@@ -30,14 +28,17 @@ import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
public class EngineFactory {
- private static ImplementationSwitch<IBatchCubingEngine> batchEngines;
- static {
- Map<Integer, String> impls = KylinConfig.getInstanceFromEnv().getJobEngines();
- batchEngines = new ImplementationSwitch<>(impls, IBatchCubingEngine.class);
- }
+ // Use thread-local because KylinConfig can be thread-local and implementation might be different among multiple threads.
+ private static ThreadLocal<ImplementationSwitch<IBatchCubingEngine>> engines = new ThreadLocal<>();
public static IBatchCubingEngine batchEngine(IEngineAware aware) {
- return batchEngines.get(aware.getEngineType());
+ ImplementationSwitch<IBatchCubingEngine> current = engines.get();
+ if (current == null) {
+ current = new ImplementationSwitch<>(KylinConfig.getInstanceFromEnv().getJobEngines(),
+ IBatchCubingEngine.class);
+ engines.set(current);
+ }
+ return current.get(aware.getEngineType());
}
/** Mark deprecated to indicate for test purpose only */
@@ -45,11 +46,11 @@ public class EngineFactory {
public static IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeDesc cubeDesc) {
return batchEngine(cubeDesc).getJoinedFlatTableDesc(cubeDesc);
}
-
+
public static IJoinedFlatTableDesc getJoinedFlatTableDesc(CubeSegment newSegment) {
return batchEngine(newSegment).getJoinedFlatTableDesc(newSegment);
}
-
+
/** Build a new cube segment, typically its time range appends to the end of current cube. */
public static DefaultChainedExecutable createBatchCubingJob(CubeSegment newSegment, String submitter) {
return batchEngine(newSegment).createBatchCubingJob(newSegment, submitter);
http://git-wip-us.apache.org/repos/asf/kylin/blob/d659bade/core-job/src/main/java/org/apache/kylin/job/SchedulerFactory.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/SchedulerFactory.java b/core-job/src/main/java/org/apache/kylin/job/SchedulerFactory.java
index 4eb76d1..1bf8942 100644
--- a/core-job/src/main/java/org/apache/kylin/job/SchedulerFactory.java
+++ b/core-job/src/main/java/org/apache/kylin/job/SchedulerFactory.java
@@ -17,27 +17,20 @@
*/
package org.apache.kylin.job;
-import java.util.Map;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ImplementationSwitch;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-/**
- */
public class SchedulerFactory {
-
- private static final Logger logger = LoggerFactory.getLogger(SchedulerFactory.class);
- private static ImplementationSwitch<Scheduler> schedulers;
-
- static {
- Map<Integer, String> impls = KylinConfig.getInstanceFromEnv().getSchedulers();
- schedulers = new ImplementationSwitch<Scheduler>(impls, Scheduler.class);
- }
+ // Use thread-local because KylinConfig can be thread-local and implementation might be different among multiple threads.
+ private static ThreadLocal<ImplementationSwitch<Scheduler>> schedulers = new ThreadLocal<>();
public static Scheduler scheduler(int schedulerType) {
- return schedulers.get(schedulerType);
+ ImplementationSwitch<Scheduler> current = schedulers.get();
+ if (current == null) {
+ current = new ImplementationSwitch<>(KylinConfig.getInstanceFromEnv().getSchedulers(), Scheduler.class);
+ schedulers.set(current);
+ }
+ return current.get(schedulerType);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/d659bade/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java b/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java
index 86f89b8..365b505 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/SourceFactory.java
@@ -19,7 +19,6 @@
package org.apache.kylin.source;
import java.util.List;
-import java.util.Map;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ImplementationSwitch;
@@ -28,19 +27,24 @@ import org.apache.kylin.metadata.model.TableDesc;
public class SourceFactory {
- private static ImplementationSwitch<ISource> sources;
- static {
- Map<Integer, String> impls = KylinConfig.getInstanceFromEnv().getSourceEngines();
- sources = new ImplementationSwitch<>(impls, ISource.class);
+ // Use thread-local because KylinConfig can be thread-local and implementation might be different among multiple threads.
+ private static ThreadLocal<ImplementationSwitch<ISource>> sources = new ThreadLocal<>();
+
+ private static ISource getSource(int sourceType) {
+ ImplementationSwitch<ISource> current = sources.get();
+ if (current == null) {
+ current = new ImplementationSwitch<>(KylinConfig.getInstanceFromEnv().getSourceEngines(), ISource.class);
+ sources.set(current);
+ }
+ return current.get(sourceType);
}
-
+
public static ISource getDefaultSource() {
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- return sources.get(config.getDefaultSource());
+ return getSource(KylinConfig.getInstanceFromEnv().getDefaultSource());
}
public static ISource getSource(ISourceAware aware) {
- return sources.get(aware.getSourceType());
+ return getSource(aware.getSourceType());
}
public static IReadableTable createReadableTable(TableDesc table) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d659bade/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
index e7dd53c..79b93fe 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageFactory.java
@@ -18,8 +18,6 @@
package org.apache.kylin.storage;
-import java.util.Map;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ImplementationSwitch;
import org.apache.kylin.metadata.model.IStorageAware;
@@ -29,14 +27,16 @@ import org.apache.kylin.metadata.realization.IRealization;
*/
public class StorageFactory {
- private static ImplementationSwitch<IStorage> storages;
- static {
- Map<Integer, String> impls = KylinConfig.getInstanceFromEnv().getStorageEngines();
- storages = new ImplementationSwitch<IStorage>(impls, IStorage.class);
- }
+ // Use thread-local because KylinConfig can be thread-local and implementation might be different among multiple threads.
+ private static ThreadLocal<ImplementationSwitch<IStorage>> storages = new ThreadLocal<>();
public static IStorage storage(IStorageAware aware) {
- return storages.get(aware.getStorageType());
+ ImplementationSwitch<IStorage> current = storages.get();
+ if (storages.get() == null) {
+ current = new ImplementationSwitch<>(KylinConfig.getInstanceFromEnv().getStorageEngines(), IStorage.class);
+ storages.set(current);
+ }
+ return current.get(aware.getStorageType());
}
public static IStorageQuery createQuery(IRealization realization) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d659bade/core-storage/src/test/java/org/apache/kylin/storage/StorageFactoryTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/StorageFactoryTest.java b/core-storage/src/test/java/org/apache/kylin/storage/StorageFactoryTest.java
new file mode 100644
index 0000000..c6ba032
--- /dev/null
+++ b/core-storage/src/test/java/org/apache/kylin/storage/StorageFactoryTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage;
+
+import java.util.Properties;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class StorageFactoryTest {
+ @Before
+ public void setUp() throws Exception {
+ Properties props = new Properties();
+ props.setProperty("kylin.storage.provider.0", MockupStorageEngine.class.getName());
+
+ KylinConfig.setKylinConfigInEnvIfMissing(props);
+ }
+
+ @Test
+ public void testSingleThread() {
+ IStorage s1 = StorageFactory.storage(new MockupStorageAware());
+ IStorage s2 = StorageFactory.storage(new MockupStorageAware());
+
+ Assert.assertSame(s1, s2);
+ }
+
+ @Test
+ public void testMultipleThread() throws InterruptedException {
+ final IStorage[] s = new IStorage[2];
+
+ // thread 1
+ Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ s[0] = StorageFactory.storage(new MockupStorageAware());
+ }
+ });
+ t.start();
+ t.join();
+
+ // thread 2
+ t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ s[1] = StorageFactory.storage(new MockupStorageAware());
+ }
+ });
+ t.start();
+ t.join();
+
+ Assert.assertNotSame(s[0], s[1]);
+ }
+
+ class MockupStorageAware implements IStorageAware {
+ @Override
+ public int getStorageType() {
+ return 0;
+ }
+ }
+
+ public static class MockupStorageEngine implements IStorage {
+
+ @Override
+ public IStorageQuery createQuery(IRealization realization) {
+ return null;
+ }
+
+ @Override
+ public <I> I adaptToBuildEngine(Class<I> engineInterface) {
+ return null;
+ }
+ }
+}