You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/09/26 13:42:09 UTC
[kylin] branch 2.6.x updated (75179ab -> c6cfbff)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 75179ab KYLIN-4121 Cleanup hive view intermediate tables after job be finished
new 7045db1 Revert "KYLIN-4046 Refine JDBC Source(source.default=8)"
new 26b0588 KYLIN-3797 Too many or filters may break Kylin server when flatting filter
new 8bd6bed KYLIN-4131 Fix Broadcaster memory leak
new 560deef KYLIN-4034 Table not in Insight when user have no access to the table
new 8494e3a minor, change kyligence nexus repository port to 80[D
new 9349e4a KYLIN-4153 Delete marker if real file not exists
new 32ad065 KYLIN-4055 cube quey and ad-hoc query return different meta info
new 699158a KYLIN-4157 When using PrepareStatement query, functions within WHERE will cause InternalErrorException
new 8083437 KYLIN-4154 Sync all events in broadcaster before closing
new 1a733c9 #KYLIN-4158, limit not push down to storage scan level if group by is an expression
new 8698f53 KYLIN-4103: Make the user string in granting operation of project is case insensitive
new c6cfbff KYLIN-3628 Fix regression bug
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/kylin/common/Closeable.java | 8 +-
.../java/org/apache/kylin/common/KylinConfig.java | 15 +
.../org/apache/kylin/common/KylinConfigBase.java | 4 +
.../org/apache/kylin/common/SourceDialect.java | 59 ----
.../common/persistence/PushdownResourceStore.java | 29 +-
.../kylin/common/persistence/ResourceStore.java | 91 ++----
.../java/org/apache/kylin/cube/CubeManager.java | 36 ++-
.../java/org/apache/kylin/job/JoinedFlatTable.java | 2 +-
.../org/apache/kylin/metadata/acl/TableACL.java | 6 +
.../kylin/metadata/cachesync/Broadcaster.java | 29 +-
.../apache/kylin/metadata/filter/TupleFilter.java | 30 +-
.../apache/kylin/metadata/model/PartitionDesc.java | 27 +-
.../org/apache/kylin/metadata/model/TblColRef.java | 18 --
.../kylin/metadata/filter/TupleFilterTest.java | 64 ++++
.../DefaultPartitionConditionBuilderTest.java | 8 +-
.../storage/gtrecord/GTCubeStorageQueryBase.java | 13 +-
.../sql_casewhen/{query55.sql => query57.sql} | 1 +
pom.xml | 5 +-
.../kylin/query/adhoc/PushDownRunnerJdbcImpl.java | 4 +-
.../apache/kylin/query/relnode/OLAPContext.java | 17 +-
.../kylin/rest/controller/QueryController.java | 4 +-
.../apache/kylin/rest/job/MetadataCleanupJob.java | 2 +-
.../apache/kylin/rest/service/AccessService.java | 4 +-
.../apache/kylin/rest/service/QueryService.java | 8 +
.../apache/kylin/rest/service/TableACLService.java | 44 +++
.../kylin/rest/controller/QueryControllerTest.java | 4 +-
.../kylin/rest/service/AccessServiceTest.java | 31 ++
.../org/apache/kylin/source/jdbc/JdbcDialect.java | 11 +-
.../org/apache/kylin/source/jdbc/JdbcExplorer.java | 17 +-
.../kylin/source/jdbc/JdbcHiveInputBase.java | 344 ++-------------------
.../apache/kylin/source/jdbc/JdbcTableReader.java | 15 +-
.../java/org/apache/kylin/source/jdbc/SqlUtil.java | 5 +-
.../source/jdbc/extensible/JdbcHiveInputBase.java | 2 +-
.../source/jdbc/metadata/DefaultJdbcMetadata.java | 6 +-
.../kylin/source/jdbc/metadata/IJdbcMetadata.java | 4 -
.../source/jdbc/metadata/JdbcMetadataFactory.java | 16 +-
.../source/jdbc/metadata/MySQLJdbcMetadata.java | 6 -
.../jdbc/metadata/SQLServerJdbcMetadata.java | 6 -
.../apache/kylin/source/jdbc/JdbcExplorerTest.java | 4 +-
.../kylin/source/jdbc/JdbcHiveInputBaseTest.java | 68 ----
.../jdbc/metadata/JdbcMetadataFactoryTest.java | 24 +-
41 files changed, 410 insertions(+), 681 deletions(-)
copy dev-support/checkstyle-apache-header.txt => core-common/src/main/java/org/apache/kylin/common/Closeable.java (90%)
delete mode 100644 core-common/src/main/java/org/apache/kylin/common/SourceDialect.java
copy kylin-it/src/test/resources/query/sql_casewhen/{query55.sql => query57.sql} (99%)
copy core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricsReporting.java => source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcDialect.java (69%)
delete mode 100644 source-jdbc/src/test/java/org/apache/kylin/source/jdbc/JdbcHiveInputBaseTest.java
copy core-common/src/test/java/org/apache/kylin/common/persistence/AutoDeleteDirectoryTest.java => source-jdbc/src/test/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactoryTest.java (58%)
[kylin] 05/12: minor, change kyligence nexus repository port to 80[D
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8494e3a2686e7d56df4ddb0ad694f3d36cdf57b4
Author: shaofengshi <sh...@apache.org>
AuthorDate: Thu Sep 5 11:12:49 2019 +0800
minor, change kyligence nexus repository port to 80[D
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 84253be..96e729e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1040,7 +1040,7 @@
<repository>
<id>kyligence</id>
<name>Kyligence Repository</name>
- <url>http://repository.kyligence.io:8081/repository/maven-public/
+ <url>http://repository.kyligence.io/repository/maven-public/
</url>
<releases>
<enabled>true</enabled>
[kylin] 11/12: KYLIN-4103: Make the user string in granting
operation of project is case insensitive
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8698f535fc907425e8d2bc9b0769262e27086f42
Author: Liu Shaohui <li...@xiaomi.com>
AuthorDate: Mon Sep 16 16:05:37 2019 +0800
KYLIN-4103: Make the user string in granting operation of project is case insensitive
---
.../apache/kylin/rest/service/AccessService.java | 4 +--
.../kylin/rest/service/AccessServiceTest.java | 31 ++++++++++++++++++++++
2 files changed, 33 insertions(+), 2 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
index dfac275..e1039e4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/AccessService.java
@@ -21,9 +21,9 @@ package org.apache.kylin.rest.service;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -422,7 +422,7 @@ public class AccessService {
}
private Map<String, Integer> getProjectPermission(String project) {
- Map<String, Integer> SidWithPermission = new HashMap<>();
+ Map<String, Integer> SidWithPermission = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
String uuid = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project).getUuid();
AclEntity ae = getAclEntity(AclEntityType.PROJECT_INSTANCE, uuid);
diff --git a/server/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
index 52f5cd2..b21abcc 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
@@ -47,6 +47,9 @@ import org.springframework.security.acls.model.Permission;
import org.springframework.security.acls.model.Sid;
import com.fasterxml.jackson.core.JsonProcessingException;
+import org.springframework.security.authentication.TestingAuthenticationToken;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.context.SecurityContextHolder;
/**
*/
@@ -183,4 +186,32 @@ public class AccessServiceTest extends ServiceTestBase {
accessService.grant(ae, AclPermission.OPERATION, sid);
}
}
+
+ @Test
+ public void testCaseInsensitiveProjectPermission() {
+ List<ProjectInstance> projects = projectService.listProjects(10000, 0);
+ assertTrue(projects.size() > 0);
+ ProjectInstance project = projects.get(0);
+ PrincipalSid sid = new PrincipalSid("ANALYST");
+ RootPersistentEntity ae = accessService.getAclEntity(PROJECT_INSTANCE, project.getUuid());
+ accessService.grant(ae, AclPermission.READ, sid);
+ Assert.assertEquals(1, accessService.getAcl(ae).getEntries().size());
+
+ Authentication admin = SecurityContextHolder.getContext().getAuthentication();
+ // Upper case username
+ Authentication analystAuth = new TestingAuthenticationToken("ANALYST", "ANALYST", "ROLE_ANALYST");
+ SecurityContextHolder.getContext().setAuthentication(analystAuth);
+ Assert.assertEquals("ANALYST", SecurityContextHolder.getContext().getAuthentication().getName());
+ Assert.assertEquals("READ", accessService.getUserPermissionInPrj(project.getName()));
+
+ // lower case username
+ analystAuth = new TestingAuthenticationToken("analyst", "ANALYST", "ROLE_ANALYST");
+ SecurityContextHolder.getContext().setAuthentication(analystAuth);
+ Assert.assertEquals("analyst", SecurityContextHolder.getContext().getAuthentication().getName());
+ Assert.assertEquals("READ", accessService.getUserPermissionInPrj(project.getName()));
+
+ SecurityContextHolder.getContext().setAuthentication(admin);
+ accessService.revokeProjectPermission("ANALYST", MetadataConstants.TYPE_USER);
+ Assert.assertEquals(0, accessService.getAcl(ae).getEntries().size());
+ }
}
[kylin] 10/12: #KYLIN-4158,
limit not push down to storage scan level if group by is an
expression
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1a733c997341811e9dd93cd89d2af01e7f2a5f6d
Author: ZhengshuaiPENG <co...@hotmail.com>
AuthorDate: Thu Sep 5 18:26:32 2019 +0800
#KYLIN-4158, limit not push down to storage scan level if group by is an expression
---
.../storage/gtrecord/GTCubeStorageQueryBase.java | 13 +++++++++---
.../test/resources/query/sql_casewhen/query57.sql | 23 ++++++++++++++++++++++
2 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 31e190c..61d5b7f 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -102,7 +102,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
return ITupleIterator.EMPTY_TUPLE_ITERATOR;
return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
- request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo, request.getContext(), sqlDigest);
+ request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo,
+ request.getContext(), sqlDigest);
}
public GTCubeStorageQueryRequest getStorageQueryRequest(StorageContext context, SQLDigest sqlDigest,
@@ -176,7 +177,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
// set limit push down
enableStorageLimitIfPossible(cuboid, groups, dynGroups, derivedPostAggregation, groupsD, filterD,
- loosenedColumnD, sqlDigest.aggregations, context);
+ loosenedColumnD, sqlDigest, context);
// set whether to aggregate results from multiple partitions
enableStreamAggregateIfBeneficial(cuboid, groupsD, context);
// check query deadline
@@ -422,7 +423,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
private void enableStorageLimitIfPossible(Cuboid cuboid, Collection<TblColRef> groups, List<TblColRef> dynGroups,
Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter,
- Set<TblColRef> loosenedColumnD, Collection<FunctionDesc> functionDescs, StorageContext context) {
+ Set<TblColRef> loosenedColumnD, SQLDigest sqlDigest, StorageContext context) {
+ Collection<FunctionDesc> functionDescs = sqlDigest.aggregations;
StorageLimitLevel storageLimitLevel = StorageLimitLevel.LIMIT_ON_SCAN;
@@ -470,6 +472,11 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
}
}
+ if (sqlDigest.groupByExpression) {
+ storageLimitLevel = StorageLimitLevel.NO_LIMIT;
+ logger.debug("storageLimitLevel set to NO_LIMIT because group by clause is an expression");
+ }
+
context.applyLimitPushDown(cubeInstance, storageLimitLevel);
}
diff --git a/kylin-it/src/test/resources/query/sql_casewhen/query57.sql b/kylin-it/src/test/resources/query/sql_casewhen/query57.sql
new file mode 100644
index 0000000..76ae649
--- /dev/null
+++ b/kylin-it/src/test/resources/query/sql_casewhen/query57.sql
@@ -0,0 +1,23 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+SELECT (CASE WHEN ("TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" IN ('Auction', 'FP-GTC')) THEN 'Auction' ELSE "TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" END) AS "LSTG_FORMAT_NAME__group_",
+ SUM("TEST_KYLIN_FACT"."PRICE") AS "sum_PRICE_ok"
+FROM "TEST_KYLIN_FACT" "TEST_KYLIN_FACT"
+GROUP BY (CASE WHEN ("TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" IN ('Auction', 'FP-GTC')) THEN 'Auction' ELSE "TEST_KYLIN_FACT"."LSTG_FORMAT_NAME" END)
+LIMIT 4
\ No newline at end of file
[kylin] 08/12: KYLIN-4157 When using PrepareStatement query,
functions within WHERE will cause InternalErrorException
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 699158af217821ea902c3db04924241b3521deb7
Author: mawu2 <ma...@cisco.com>
AuthorDate: Wed Sep 4 20:47:19 2019 +0800
KYLIN-4157 When using PrepareStatement query, functions within WHERE will cause InternalErrorException
---
.../org/apache/kylin/query/relnode/OLAPContext.java | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index b8ddd02..58f6e1e 100755
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import org.apache.calcite.DataContext;
@@ -316,8 +317,7 @@ public class OLAPContext {
Object value = dataContext.get(variable);
if (value != null) {
String str = value.toString();
- if (compFilter.getColumn().getType().isDateTimeFamily())
- str = String.valueOf(DateFormat.stringToMillis(str));
+ str = transferDateTimeColumnToMillis(compFilter, str);
compFilter.clearPreviousVariableValues(variable);
compFilter.bindVariable(variable, str);
}
@@ -325,6 +325,19 @@ public class OLAPContext {
}
}
}
+
+ private String transferDateTimeColumnToMillis(CompareTupleFilter compFilter, String value) {
+ TblColRef column = compFilter.getColumn();
+ // To fix KYLIN-4157, when using PrepareStatement query, functions within WHERE will cause InternalErrorException
+ if (Objects.isNull(column)){
+ return value;
+ }
+
+ if (column.getType().isDateTimeFamily()){
+ value = String.valueOf(DateFormat.stringToMillis(value));
+ }
+ return value;
+ }
// ============================================================================
public interface IAccessController {
[kylin] 02/12: KYLIN-3797 Too many or filters may break Kylin
server when flatting filter
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 26b0588fd290bb0ee0fb49f4dc9b7b0931da31aa
Author: Ma,Gang <ga...@ebay.com>
AuthorDate: Wed Jan 30 15:40:11 2019 +0800
KYLIN-3797 Too many or filters may break Kylin server when flatting filter
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++
.../apache/kylin/metadata/filter/TupleFilter.java | 30 ++++++++--
.../kylin/metadata/filter/TupleFilterTest.java | 64 ++++++++++++++++++++++
3 files changed, 92 insertions(+), 6 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 91c86b5..aa8626f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1706,6 +1706,10 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(this.getOptional("kylin.query.cache-signature-enabled", FALSE));
}
+ public int getFlatFilterMaxChildrenSize() {
+ return Integer.parseInt(this.getOptional("kylin.query.flat-filter-max-children", "500000"));
+ }
+
// ============================================================================
// SERVER
// ============================================================================
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
index 672aba0..019960e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/TupleFilter.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
import org.slf4j.Logger;
@@ -215,10 +216,19 @@ public abstract class TupleFilter {
* @return
*/
public TupleFilter flatFilter() {
- return flattenInternal(this);
+ return flatFilter(KylinConfig.getInstanceFromEnv().getFlatFilterMaxChildrenSize());
}
- private TupleFilter flattenInternal(TupleFilter filter) {
+ /**
+ * throws IllegalStateException when the flat children exceed the maxFlatChildrenSize
+ * @param maxFlatChildrenSize
+ * @return
+ */
+ public TupleFilter flatFilter(int maxFlatChildrenSize) {
+ return flattenInternal(this, maxFlatChildrenSize);
+ }
+
+ private TupleFilter flattenInternal(TupleFilter filter, int maxFlatChildrenSize) {
TupleFilter flatFilter = null;
if (!(filter instanceof LogicalTupleFilter)) {
flatFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
@@ -231,7 +241,7 @@ public abstract class TupleFilter {
List<TupleFilter> andChildren = new LinkedList<TupleFilter>();
List<TupleFilter> orChildren = new LinkedList<TupleFilter>();
for (TupleFilter child : filter.getChildren()) {
- TupleFilter flatChild = flattenInternal(child);
+ TupleFilter flatChild = flattenInternal(child, maxFlatChildrenSize);
FilterOperatorEnum childOp = flatChild.getOperator();
if (childOp == FilterOperatorEnum.AND) {
andChildren.add(flatChild);
@@ -249,7 +259,7 @@ public abstract class TupleFilter {
flatFilter.addChildren(andChild.getChildren());
}
if (!orChildren.isEmpty()) {
- List<TupleFilter> fullAndFilters = cartesianProduct(orChildren, flatFilter);
+ List<TupleFilter> fullAndFilters = cartesianProduct(orChildren, flatFilter, maxFlatChildrenSize);
flatFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
flatFilter.addChildren(fullAndFilters);
}
@@ -262,14 +272,18 @@ public abstract class TupleFilter {
} else if (op == FilterOperatorEnum.NOT) {
assert (filter.children.size() == 1);
TupleFilter reverse = filter.children.get(0).reverse();
- flatFilter = flattenInternal(reverse);
+ flatFilter = flattenInternal(reverse, maxFlatChildrenSize);
} else {
throw new IllegalStateException("Filter is " + filter);
}
+ if (flatFilter.getChildren() != null && flatFilter.getChildren().size() > maxFlatChildrenSize) {
+ throw new IllegalStateException("the filter is too large after do the flat, size="
+ + flatFilter.getChildren().size());
+ }
return flatFilter;
}
- private List<TupleFilter> cartesianProduct(List<TupleFilter> leftOrFilters, TupleFilter partialAndFilter) {
+ private List<TupleFilter> cartesianProduct(List<TupleFilter> leftOrFilters, TupleFilter partialAndFilter, int maxFlatChildrenSize) {
List<TupleFilter> oldProductFilters = new LinkedList<TupleFilter>();
oldProductFilters.add(partialAndFilter);
for (TupleFilter orFilter : leftOrFilters) {
@@ -279,6 +293,10 @@ public abstract class TupleFilter {
TupleFilter fullAndFilter = productFilter.copy();
fullAndFilter.addChildren(orChildFilter.getChildren());
newProductFilters.add(fullAndFilter);
+ if (newProductFilters.size() > maxFlatChildrenSize) {
+ throw new IllegalStateException("the filter is too large after do the flat, size="
+ + newProductFilters.size());
+ }
}
}
oldProductFilters = newProductFilters;
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
index 95609eb..6a3843b 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/filter/TupleFilterTest.java
@@ -28,6 +28,7 @@ import java.util.Set;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.TblColRef.InnerDataTypeEnum;
import org.junit.Assert;
import org.junit.Test;
@@ -138,5 +139,68 @@ public class TupleFilterTest {
Assert.assertEquals(Sets.newHashSet(compFilter2), compFilter2.findMustTrueCompareFilters());
}
+
+ @Test
+ public void flatFilterTest() {
+ TupleFilter topAndFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ topAndFilter.addChild(createEQFilter("c1", "v1"));
+ TupleFilter orFilter1 = new LogicalTupleFilter(FilterOperatorEnum.OR);
+ TupleFilter andFilter11 = new LogicalTupleFilter(FilterOperatorEnum.AND);
+ andFilter11.addChild(createEQFilter("c2", "v2"));
+ andFilter11.addChild(createEQFilter("c3", "v3"));
+
+ TupleFilter andFilter12 = new LogicalTupleFilter(FilterOperatorEnum.AND);
+ andFilter12.addChild(createEQFilter("c2", "v21"));
+ andFilter12.addChild(createEQFilter("c3", "v31"));
+
+ orFilter1.addChild(andFilter11);
+ orFilter1.addChild(andFilter12);
+
+ TupleFilter orFilter2 = new LogicalTupleFilter(FilterOperatorEnum.OR);
+ TupleFilter andFilter21 = new LogicalTupleFilter(FilterOperatorEnum.AND);
+ andFilter21.addChild(createEQFilter("c4", "v4"));
+ andFilter21.addChild(createEQFilter("c5", "v5"));
+
+ TupleFilter andFilter22 = new LogicalTupleFilter(FilterOperatorEnum.AND);
+ andFilter22.addChild(createEQFilter("c4", "v41"));
+ andFilter22.addChild(createEQFilter("c5", "v51"));
+
+ TupleFilter andFilter23 = new LogicalTupleFilter(FilterOperatorEnum.AND);
+ andFilter23.addChild(createEQFilter("c4", "v42"));
+ andFilter23.addChild(createEQFilter("c5", "v52"));
+
+ orFilter2.addChild(andFilter21);
+ orFilter2.addChild(andFilter22);
+ orFilter2.addChild(andFilter23);
+
+ topAndFilter.addChild(orFilter1);
+ topAndFilter.addChild(orFilter2);
+
+ TupleFilter flatFilter = topAndFilter.flatFilter(500000);
+ Assert.assertEquals(6, flatFilter.children.size());
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void flatFilterTooFatTest() {
+ TupleFilter topAndFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ for (int i = 0; i < 3; i++) {
+ TupleFilter orFilter = new LogicalTupleFilter(FilterOperatorEnum.OR);
+ String col = "col-" + i;
+ for (int j = 0; j < 100; j++) {
+ orFilter.addChild(createEQFilter(col, String.valueOf(j)));
+ }
+ topAndFilter.addChild(orFilter);
+ }
+ TupleFilter flatFilter = topAndFilter.flatFilter(500000);
+ System.out.println(flatFilter);
+ }
+
+ private TupleFilter createEQFilter(String colName, String colVal) {
+ CompareTupleFilter compareTupleFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
+ compareTupleFilter
+ .addChild(new ColumnTupleFilter(TblColRef.newInnerColumn(colName, InnerDataTypeEnum.LITERAL)));
+ compareTupleFilter.addChild(new ConstantTupleFilter(colVal));
+ return compareTupleFilter;
+ }
}
[kylin] 09/12: KYLIN-4154 Sync all events in broadcaster before
closing
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 80834373edee373d25f87d53e4a2530c34c20a4b
Author: chao long <wa...@qq.com>
AuthorDate: Fri Sep 6 17:11:26 2019 +0800
KYLIN-4154 Sync all events in broadcaster before closing
---
.../apache/kylin/metadata/cachesync/Broadcaster.java | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 7e6ee7c..ef774a6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -24,6 +24,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
@@ -87,6 +88,7 @@ public class Broadcaster implements Closeable {
private BlockingDeque<BroadcastEvent> broadcastEvents = new LinkedBlockingDeque<>();
private Map<String, List<Listener>> listenerMap = Maps.newConcurrentMap();
private AtomicLong counter = new AtomicLong(); // a counter for testing purpose
+ private final BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>();
private Broadcaster(final KylinConfig config) {
this.config = config;
@@ -102,7 +104,7 @@ public class Broadcaster implements Closeable {
int corePoolSize = (nodes == null || nodes.length < 1)? 1 : nodes.length;
int maximumPoolSize = (nodes == null || nodes.length < 1)? 10 : nodes.length * 2;
this.announceThreadPool = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, 60L, TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>(), new DaemonThreadFactory());
+ workQueue, new DaemonThreadFactory());
announceMainLoop.execute(new Runnable() {
@Override
@@ -156,7 +158,7 @@ public class Broadcaster implements Closeable {
@Override
public void close() {
- stopAnnounce();
+ new Thread(this::stopAnnounce).start();
}
private SyncErrorHandler getSyncErrorHandler(KylinConfig config) {
@@ -172,7 +174,20 @@ public class Broadcaster implements Closeable {
}
public void stopAnnounce() {
+ synchronized (workQueue) {
+ while (!workQueue.isEmpty() || !broadcastEvents.isEmpty()){
+ try {
+ workQueue.wait(100);
+ } catch (InterruptedException e) {
+ logger.warn("InterruptedException is caught when waiting workQueue empty.", e);
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ logger.info("AnnounceThreadPool shutdown.");
announceThreadPool.shutdownNow();
+ logger.info("AnnounceMainLoop shutdown.");
announceMainLoop.shutdownNow();
}
[kylin] 06/12: KYLIN-4153 Delete marker if real file not exists
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9349e4adf50e20b79e3675dd798009039041defb
Author: XiaoxiangYu <hi...@126.com>
AuthorDate: Sun Sep 1 23:15:27 2019 +0800
KYLIN-4153 Delete marker if real file not exists
---
.../common/persistence/PushdownResourceStore.java | 29 ++++---
.../kylin/common/persistence/ResourceStore.java | 91 ++++++++--------------
.../apache/kylin/rest/job/MetadataCleanupJob.java | 2 +-
3 files changed, 51 insertions(+), 71 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/PushdownResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/PushdownResourceStore.java
index 7cb8ca6..587b691 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/PushdownResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/PushdownResourceStore.java
@@ -39,21 +39,21 @@ import org.slf4j.LoggerFactory;
* - The big resource is saved as HDFS file according to its resource path.
* - Method like checkAndPut() does not work on big resource like such, because HDFS lack of transaction support.
*/
-abstract public class PushdownResourceStore extends ResourceStore {
- private static final Logger logger = LoggerFactory.getLogger(HDFSResourceStore.class);
+public abstract class PushdownResourceStore extends ResourceStore {
+ private static final Logger logger = LoggerFactory.getLogger(PushdownResourceStore.class);
protected PushdownResourceStore(KylinConfig kylinConfig) {
super(kylinConfig);
}
- final protected void putResourceImpl(String resPath, ContentWriter content, long ts) throws IOException {
+ protected final void putResourceImpl(String resPath, ContentWriter content, long ts) throws IOException {
if (content.isBigContent())
putBigResource(resPath, content, ts);
else
putSmallResource(resPath, content, ts);
}
- abstract protected void putSmallResource(String resPath, ContentWriter content, long ts) throws IOException;
+ protected abstract void putSmallResource(String resPath, ContentWriter content, long ts) throws IOException;
final void putBigResource(String resPath, ContentWriter content, long newTS) throws IOException {
@@ -84,12 +84,14 @@ abstract public class PushdownResourceStore extends ResourceStore {
Path backPath;
boolean hasOldFile;
boolean hasRollback = false;
+ String resPathStr;
private RollbackablePushdown(String resPath, ContentWriter content) throws IOException {
int salt = System.identityHashCode(resPath) + System.identityHashCode(content);
tempPath = pushdownPath(resPath + ".temp." + salt);
realPath = pushdownPath(resPath);
backPath = pushdownPath(resPath + ".orig." + salt);
+ resPathStr = resPath;
fs = pushdownFS();
if (fs.exists(tempPath))
@@ -136,9 +138,12 @@ abstract public class PushdownResourceStore extends ResourceStore {
if (fs.exists(realPath))
fs.delete(realPath, true);
- if (hasOldFile)
+ if (hasOldFile) {
fs.rename(backPath, realPath);
-
+ } else {
+ logger.warn("Try delete empty entry {}", resPathStr);
+ deleteResourceImpl(resPathStr);
+ }
} catch (IOException ex2) {
logger.error("Rollback failed", ex2);
}
@@ -167,23 +172,25 @@ abstract public class PushdownResourceStore extends ResourceStore {
try {
Path p = pushdownPath(resPath);
FileSystem fs = pushdownFS();
- if (fs.exists(p))
+ if (fs.exists(p)) {
return fs.open(p);
- else
+ } else {
+ logger.error("Marker exists but real file not found, delete marker.");
+ deleteResourceImpl(resPath);
throw new FileNotFoundException(p.toString() + " (FS: " + fs + ")");
-
+ }
} catch (Exception ex) {
throw new IOException("Failed to read big resource " + resPath, ex);
}
}
- abstract protected String pushdownRootPath();
+ protected abstract String pushdownRootPath();
protected FileSystem pushdownFS() {
return HadoopUtil.getFileSystem(new Path(kylinConfig.getMetastoreBigCellHdfsDirectory()));
}
- final protected Path pushdownPath(String resPath) {
+ protected final Path pushdownPath(String resPath) {
Path p = new Path(pushdownRootPath() + resPath);
return Path.getPathWithoutSchemeAndAuthority(p);
}
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index 3eeeb69..6568279 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -136,7 +136,7 @@ abstract public class ResourceStore {
StringEntity.serializer);
}
StringEntity entity = getResource(ResourceStore.METASTORE_UUID_TAG, StringEntity.serializer);
- return entity.toString();
+ return entity == null ? "":entity.toString();
}
/**
@@ -286,12 +286,7 @@ abstract public class ResourceStore {
private RawResource getResourceWithRetry(final String resPath) throws IOException {
ExponentialBackoffRetry retry = new ExponentialBackoffRetry(this);
- return retry.doWithRetry(new Callable<RawResource>() {
- @Override
- public RawResource call() throws IOException {
- return getResourceImpl(resPath);
- }
- });
+ return retry.doWithRetry(() -> getResourceImpl(resPath));
}
final public long getResourceTimestamp(String resPath) throws IOException {
@@ -307,12 +302,7 @@ abstract public class ResourceStore {
final String path = norm(resPath);
ExponentialBackoffRetry retry = new ExponentialBackoffRetry(this);
- return retry.doWithRetry(new Callable<Long>() {
- @Override
- public Long call() throws IOException {
- return getResourceTimestampImpl(path);
- }
- });
+ return retry.doWithRetry(() -> getResourceTimestampImpl(path));
}
/**
@@ -364,19 +354,16 @@ abstract public class ResourceStore {
protected void putResourceWithRetry(final String resPath, final ContentWriter content, final long ts)
throws IOException {
ExponentialBackoffRetry retry = new ExponentialBackoffRetry(this);
- retry.doWithRetry(new Callable() {
- @Override
- public Object call() throws IOException {
- putResourceImpl(resPath, content, ts);
- return null;
- }
+ retry.doWithRetry(() -> {
+ putResourceImpl(resPath, content, ts);
+ return null;
});
}
/**
* check & set, overwrite a resource
*/
- final public <T extends RootPersistentEntity> void checkAndPutResource(String resPath, T obj,
+ public final <T extends RootPersistentEntity> void checkAndPutResource(String resPath, T obj,
Serializer<T> serializer) throws IOException, WriteConflictException {
checkAndPutResource(resPath, obj, System.currentTimeMillis(), serializer);
}
@@ -384,7 +371,7 @@ abstract public class ResourceStore {
/**
* check & set, overwrite a resource
*/
- final public <T extends RootPersistentEntity> void checkAndPutResource(String resPath, T obj, long newTS,
+ public final <T extends RootPersistentEntity> void checkAndPutResource(String resPath, T obj, long newTS,
Serializer<T> serializer) throws IOException, WriteConflictException {
resPath = norm(resPath);
@@ -413,7 +400,7 @@ abstract public class ResourceStore {
*
* @return a confirmed TS, as some store may lose timestamp precision.
*/
- final public long checkAndPutResource(String resPath, byte[] content, long oldTS, long newTS)
+ public final long checkAndPutResource(String resPath, byte[] content, long oldTS, long newTS)
throws IOException, WriteConflictException {
return checkAndPutResourceCheckpoint(norm(resPath), content, oldTS, newTS);
}
@@ -427,24 +414,19 @@ abstract public class ResourceStore {
/**
* checks old timestamp when overwriting existing
*/
- abstract protected long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS)
+ protected abstract long checkAndPutResourceImpl(String resPath, byte[] content, long oldTS, long newTS)
throws IOException, WriteConflictException;
private long checkAndPutResourceWithRetry(final String resPath, final byte[] content, final long oldTS,
final long newTS) throws IOException, WriteConflictException {
ExponentialBackoffRetry retry = new ExponentialBackoffRetry(this);
- return retry.doWithRetry(new Callable<Long>() {
- @Override
- public Long call() throws IOException {
- return checkAndPutResourceImpl(resPath, content, oldTS, newTS);
- }
- });
+ return retry.doWithRetry(() -> checkAndPutResourceImpl(resPath, content, oldTS, newTS));
}
/**
* update resource timestamp to timestamp
*/
- final public void updateTimestamp(String resPath, long timestamp) throws IOException {
+ public final void updateTimestamp(String resPath, long timestamp) throws IOException {
logger.trace("Updating resource: {} with timestamp {} (Store {})", resPath, timestamp,
kylinConfig.getMetadataUrl());
updateTimestampCheckPoint(norm(resPath), timestamp);
@@ -457,21 +439,18 @@ abstract public class ResourceStore {
private void updateTimestampWithRetry(final String resPath, final long timestamp) throws IOException {
ExponentialBackoffRetry retry = new ExponentialBackoffRetry(this);
- retry.doWithRetry(new Callable() {
- @Override
- public Object call() throws IOException {
- updateTimestampImpl(resPath, timestamp);
- return null;
- }
+ retry.doWithRetry(() -> {
+ updateTimestampImpl(resPath, timestamp);
+ return null;
});
}
- abstract protected void updateTimestampImpl(String resPath, long timestamp) throws IOException;
+ protected abstract void updateTimestampImpl(String resPath, long timestamp) throws IOException;
/**
* delete a resource, does nothing on a folder
*/
- final public void deleteResource(String resPath) throws IOException {
+ public final void deleteResource(String resPath) throws IOException {
logger.trace("Deleting resource {} (Store {})", resPath, kylinConfig.getMetadataUrl());
deleteResourceCheckpoint(norm(resPath));
}
@@ -482,7 +461,7 @@ abstract public class ResourceStore {
* throw an IOException when the resource lastModified >= timestamp + 1000
* See https://issues.apache.org/jira/browse/KYLIN-4030
*/
- final public void deleteResource(String resPath, long timestamp) throws IOException {
+ public final void deleteResource(String resPath, long timestamp) throws IOException {
logger.trace("Deleting resource {} within timestamp {} (Store {})", resPath, timestamp,
kylinConfig.getMetadataUrl());
deleteResourceCheckpoint(norm(resPath), timestamp);
@@ -498,29 +477,23 @@ abstract public class ResourceStore {
deleteResourceWithRetry(resPath, timestamp);
}
- abstract protected void deleteResourceImpl(String resPath) throws IOException;
+ protected abstract void deleteResourceImpl(String resPath) throws IOException;
- abstract protected void deleteResourceImpl(String resPath, long timestamp) throws IOException;
+ protected abstract void deleteResourceImpl(String resPath, long timestamp) throws IOException;
private void deleteResourceWithRetry(final String resPath) throws IOException {
ExponentialBackoffRetry retry = new ExponentialBackoffRetry(this);
- retry.doWithRetry(new Callable() {
- @Override
- public Object call() throws IOException {
- deleteResourceImpl(resPath);
- return null;
- }
+ retry.doWithRetry(() -> {
+ deleteResourceImpl(resPath);
+ return null;
});
}
private void deleteResourceWithRetry(final String resPath, final long timestamp) throws IOException {
ExponentialBackoffRetry retry = new ExponentialBackoffRetry(this);
- retry.doWithRetry(new Callable() {
- @Override
- public Object call() throws IOException {
- deleteResourceImpl(resPath, timestamp);
- return null;
- }
+ retry.doWithRetry(() -> {
+ deleteResourceImpl(resPath, timestamp);
+ return null;
});
}
@@ -574,11 +547,11 @@ abstract public class ResourceStore {
/**
* get a readable string of a resource path
*/
- final public String getReadableResourcePath(String resPath) {
+ public final String getReadableResourcePath(String resPath) {
return getReadableResourcePathImpl(norm(resPath));
}
- abstract protected String getReadableResourcePathImpl(String resPath);
+ protected abstract String getReadableResourcePathImpl(String resPath);
private String norm(String resPath) {
resPath = resPath.trim();
@@ -727,7 +700,7 @@ abstract public class ResourceStore {
* Visit all resource under a folder (optionally recursively), without loading the content of resource.
* Low level API, DON'T support ExponentialBackoffRetry, caller should do necessary retry
*/
- final public void visitFolder(String folderPath, boolean recursive, Visitor visitor) throws IOException {
+ public final void visitFolder(String folderPath, boolean recursive, Visitor visitor) throws IOException {
visitFolderInner(folderPath, recursive, null, false, visitor);
}
@@ -735,7 +708,7 @@ abstract public class ResourceStore {
* Visit all resource under a folder (optionally recursively), without loading the content of resource.
* Low level API, DON'T support ExponentialBackoffRetry, caller should do necessary retry
*/
- final public void visitFolder(String folderPath, boolean recursive, VisitFilter filter, Visitor visitor)
+ public final void visitFolder(String folderPath, boolean recursive, VisitFilter filter, Visitor visitor)
throws IOException {
visitFolderInner(folderPath, recursive, filter, false, visitor);
}
@@ -744,7 +717,7 @@ abstract public class ResourceStore {
* Visit all resource and their content under a folder (optionally recursively).
* Low level API, DON'T support ExponentialBackoffRetry, caller should do necessary retry
*/
- final public void visitFolderAndContent(String folderPath, boolean recursive, VisitFilter filter, Visitor visitor)
+ public final void visitFolderAndContent(String folderPath, boolean recursive, VisitFilter filter, Visitor visitor)
throws IOException {
visitFolderInner(folderPath, recursive, filter, true, visitor);
}
@@ -774,7 +747,7 @@ abstract public class ResourceStore {
*
* NOTE: Broken content exception should be wrapped by RawResource, and return to caller to decide how to handle.
*/
- abstract protected void visitFolderImpl(String folderPath, boolean recursive, VisitFilter filter,
+ protected abstract void visitFolderImpl(String folderPath, boolean recursive, VisitFilter filter,
boolean loadContent, Visitor visitor) throws IOException;
public static String dumpResources(KylinConfig kylinConfig, Collection<String> dumpList) throws IOException {
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
index 17eebd6..cb56c25 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/MetadataCleanupJob.java
@@ -242,7 +242,7 @@ public class MetadataCleanupJob {
}
private NavigableSet<String> noNull(NavigableSet<String> list) {
- return (list == null) ? new TreeSet<String>() : list;
+ return (list == null) ? new TreeSet<>() : list;
}
private long getTimestamp(String resPath) {
[kylin] 04/12: KYLIN-4034 Table not in Insight when user have no
access to the table
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 560deef12ea6173644e8250958e5f17d0ee0fcab
Author: yaqian.zhang <59...@qq.com>
AuthorDate: Fri Aug 30 20:24:27 2019 +0800
KYLIN-4034 Table not in Insight when user have no access to the table
---
.../org/apache/kylin/metadata/acl/TableACL.java | 6 +++
.../kylin/rest/controller/QueryController.java | 4 +-
.../apache/kylin/rest/service/QueryService.java | 8 ++++
.../apache/kylin/rest/service/TableACLService.java | 44 ++++++++++++++++++++++
.../kylin/rest/controller/QueryControllerTest.java | 4 +-
5 files changed, 63 insertions(+), 3 deletions(-)
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
index 57ebb61..b110064 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/acl/TableACL.java
@@ -34,6 +34,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.NONE,
@@ -86,6 +87,11 @@ public class TableACL extends RootPersistentEntity {
return currentEntry(type).getCanAccessList(table, allIdentifiers);
}
+ public Set<String> getBlockedTablesByUser(String username, String type) {
+ return currentEntry(type).get(username) == null ? Sets.<String>newHashSet()
+ : currentEntry(type).get(username).getTables();
+ }
+
public TableACL add(String name, String table, String type) {
currentEntry(type).add(name, table);
return this;
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index b89772b..6b56e91 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -186,9 +186,9 @@ public class QueryController extends BasicController {
@RequestMapping(value = "/tables_and_columns", method = RequestMethod.GET, produces = { "application/json" })
@ResponseBody
- public List<TableMeta> getMetadata(MetaRequest metaRequest) {
+ public List<TableMeta> getMetadata(MetaRequest metaRequest) throws IOException {
try {
- return queryService.getMetadata(metaRequest.getProject());
+ return queryService.getMetadataFilterByUser(metaRequest.getProject());
} catch (SQLException e) {
throw new InternalErrorException(e.getLocalizedMessage(), e);
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index da4fcb6..bf8082e 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -156,6 +156,10 @@ public class QueryService extends BasicService {
private ModelService modelService;
@Autowired
+ @Qualifier("TableAclService")
+ private TableACLService tableAclService;
+
+ @Autowired
private AclEvaluate aclEvaluate;
private GenericKeyedObjectPool<PreparedContextKey, PreparedContext> preparedContextPool;
@@ -195,6 +199,10 @@ public class QueryService extends BasicService {
Preconditions.checkNotNull(cacheManager, "cacheManager is not injected yet");
}
+ public List<TableMeta> getMetadataFilterByUser(String project) throws SQLException, IOException {
+ return tableAclService.filterTableMetasByAcl(getMetadata(project), project);
+ }
+
public List<TableMeta> getMetadata(String project) throws SQLException {
return getMetadata(getCubeManager(), project);
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java
index c054ba0..964e9d6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableACLService.java
@@ -18,16 +18,24 @@
package org.apache.kylin.rest.service;
+import static org.apache.kylin.metadata.MetadataConstants.TYPE_USER;
+
import java.io.IOException;
import java.util.List;
import java.util.Set;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.metadata.acl.TableACL;
import org.apache.kylin.rest.util.AclEvaluate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
+import org.springframework.security.core.context.SecurityContextHolder;
+
+import com.google.common.collect.Lists;
@Component("TableAclService")
public class TableACLService extends BasicService {
@@ -74,4 +82,40 @@ public class TableACLService extends BasicService {
aclEvaluate.checkProjectAdminPermission(project);
getTableACLManager().deleteTableACLByTbl(project, table);
}
+
+ public List<TableMeta> filterTableMetasByAcl(List<TableMeta> tableMeta, String project) throws IOException {
+ return filterByAcl(tableMeta, project, new AclFilter<TableMeta>() {
+ @Override
+ public boolean filter(TableMeta table, Set<String> blockedTables) {
+ String identity = table.getTABLE_SCHEM() + "." + table.getTABLE_NAME();
+ return !blockedTables.contains(identity);
+ }
+ });
+ }
+
+ private interface AclFilter<T> {
+ boolean filter(T table, Set<String> blockedTables);
+ }
+
+ private <T> List<T> filterByAcl(List<T> tables, String project, AclFilter filter) throws IOException {
+ ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
+
+ if (aclEvaluate.hasProjectAdminPermission(projectManager.getProject(project))) {
+ return tables;
+ }
+
+ String username = SecurityContextHolder.getContext().getAuthentication().getName();
+ Set<String> blockedTables = getBlockedTablesByUser(project, username, TYPE_USER);
+ List<T> result = Lists.newArrayList();
+ for (T table : tables) {
+ if (filter.filter(table, blockedTables)) {
+ result.add(table);
+ }
+ }
+ return result;
+ }
+
+ private Set<String> getBlockedTablesByUser(String project, String username, String type) throws IOException {
+ return getTableACLByProject(project).getBlockedTablesByUser(username, type);
+ }
}
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
index 2225096..d18c2ed 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/QueryControllerTest.java
@@ -33,6 +33,8 @@ import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.cache.CacheManager;
+import java.io.IOException;
+
/**
* @author xduo
*/
@@ -77,7 +79,7 @@ public class QueryControllerTest extends ServiceTestBase {
}
@Test
- public void testGetMetadata() {
+ public void testGetMetadata() throws IOException {
queryController.getMetadata(new MetaRequest(ProjectInstance.DEFAULT_PROJECT_NAME));
}
[kylin] 01/12: Revert "KYLIN-4046 Refine JDBC
Source(source.default=8)"
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 7045db1b5fa35f25f43378b9cb6a2b77a0e98944
Author: nichunen <ni...@apache.org>
AuthorDate: Fri Aug 16 15:36:06 2019 +0800
Revert "KYLIN-4046 Refine JDBC Source(source.default=8)"
This reverts commit 37ee3a81603baf7b1d8252736bf94f037140320c.
---
.../java/org/apache/kylin/job/JoinedFlatTable.java | 2 +-
.../apache/kylin/metadata/model/PartitionDesc.java | 27 +-
.../org/apache/kylin/metadata/model/TblColRef.java | 18 --
.../DefaultPartitionConditionBuilderTest.java | 8 +-
pom.xml | 3 -
.../JdbcMetadataFactory.java => JdbcDialect.java} | 26 +-
.../org/apache/kylin/source/jdbc/JdbcExplorer.java | 17 +-
.../kylin/source/jdbc/JdbcHiveInputBase.java | 344 ++-------------------
.../apache/kylin/source/jdbc/JdbcTableReader.java | 15 +-
.../java/org/apache/kylin/source/jdbc/SqlUtil.java | 5 +-
.../source/jdbc/extensible/JdbcHiveInputBase.java | 2 +-
.../source/jdbc/metadata/DefaultJdbcMetadata.java | 6 +-
.../kylin/source/jdbc/metadata/IJdbcMetadata.java | 4 -
.../source/jdbc/metadata/JdbcMetadataFactory.java | 16 +-
.../source/jdbc/metadata/MySQLJdbcMetadata.java | 6 -
.../jdbc/metadata/SQLServerJdbcMetadata.java | 6 -
.../apache/kylin/source/jdbc/JdbcExplorerTest.java | 4 +-
.../kylin/source/jdbc/JdbcHiveInputBaseTest.java | 68 ----
.../jdbc/metadata/JdbcMetadataFactoryTest.java | 56 +---
19 files changed, 81 insertions(+), 552 deletions(-)
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 4281885..0d1cafb 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -241,7 +241,7 @@ public class JoinedFlatTable {
if (segRange != null && !segRange.isInfinite()) {
whereBuilder.append(" AND (");
String quotedPartitionCond = quoteIdentifierInSqlExpr(flatDesc,
- partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange, null));
+ partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange));
whereBuilder.append(quotedPartitionCond);
whereBuilder.append(")" + sep);
}
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index f93996e..56ededb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -20,7 +20,6 @@ package org.apache.kylin.metadata.model;
import java.io.Serializable;
import java.util.Locale;
-import java.util.function.Function;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.util.ClassUtil;
@@ -185,26 +184,19 @@ public class PartitionDesc implements Serializable {
// ============================================================================
public static interface IPartitionConditionBuilder {
- String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange, Function<TblColRef, String> quoteFunc);
+ String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange);
}
public static class DefaultPartitionConditionBuilder implements IPartitionConditionBuilder, Serializable {
@Override
- public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange, Function<TblColRef, String> quoteFunc) {
+ public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange) {
long startInclusive = (Long) segRange.start.v;
long endExclusive = (Long) segRange.end.v;
TblColRef partitionDateColumn = partDesc.getPartitionDateColumnRef();
TblColRef partitionTimeColumn = partDesc.getPartitionTimeColumnRef();
- if (partitionDateColumn != null) {
- partitionDateColumn.setQuotedFunc(quoteFunc);
- }
- if (partitionTimeColumn != null) {
- partitionTimeColumn.setQuotedFunc(quoteFunc);
- }
-
StringBuilder builder = new StringBuilder();
if (partDesc.partitionColumnIsYmdInt()) {
@@ -232,7 +224,7 @@ public class PartitionDesc implements Serializable {
private static void buildSingleColumnRangeCondAsTimeMillis(StringBuilder builder, TblColRef partitionColumn,
long startInclusive, long endExclusive) {
- String partitionColumnName = partitionColumn.getQuotedIdentity();
+ String partitionColumnName = partitionColumn.getIdentity();
builder.append(partitionColumnName + " >= " + startInclusive);
builder.append(" AND ");
builder.append(partitionColumnName + " < " + endExclusive);
@@ -240,7 +232,7 @@ public class PartitionDesc implements Serializable {
private static void buildSingleColumnRangeCondAsYmdInt(StringBuilder builder, TblColRef partitionColumn,
long startInclusive, long endExclusive, String partitionColumnDateFormat) {
- String partitionColumnName = partitionColumn.getQuotedIdentity();
+ String partitionColumnName = partitionColumn.getIdentity();
builder.append(partitionColumnName + " >= "
+ DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat));
builder.append(" AND ");
@@ -250,7 +242,7 @@ public class PartitionDesc implements Serializable {
private static void buildSingleColumnRangeCondition(StringBuilder builder, TblColRef partitionColumn,
long startInclusive, long endExclusive, String partitionColumnDateFormat) {
- String partitionColumnName = partitionColumn.getQuotedIdentity();
+ String partitionColumnName = partitionColumn.getIdentity();
if (endExclusive <= startInclusive) {
builder.append("1=0");
@@ -275,8 +267,8 @@ public class PartitionDesc implements Serializable {
private static void buildMultipleColumnRangeCondition(StringBuilder builder, TblColRef partitionDateColumn,
TblColRef partitionTimeColumn, long startInclusive, long endExclusive, String partitionColumnDateFormat,
String partitionColumnTimeFormat, boolean partitionDateColumnIsYmdInt) {
- String partitionDateColumnName = partitionDateColumn.getQuotedIdentity();
- String partitionTimeColumnName = partitionTimeColumn.getQuotedIdentity();
+ String partitionDateColumnName = partitionDateColumn.getIdentity();
+ String partitionTimeColumnName = partitionTimeColumn.getIdentity();
String singleQuotation = partitionDateColumnIsYmdInt ? "" : "'";
builder.append("(");
builder.append("(");
@@ -316,14 +308,11 @@ public class PartitionDesc implements Serializable {
public static class YearMonthDayPartitionConditionBuilder implements IPartitionConditionBuilder {
@Override
- public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange, Function<TblColRef, String> func) {
+ public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange) {
long startInclusive = (Long) segRange.start.v;
long endExclusive = (Long) segRange.end.v;
TblColRef partitionColumn = partDesc.getPartitionDateColumnRef();
- if (partitionColumn != null) {
- partitionColumn.setQuotedFunc(func);
- }
String tableAlias = partitionColumn.getTableAlias();
String concatField = String.format(Locale.ROOT, "CONCAT(%s.YEAR,'-',%s.MONTH,'-',%s.DAY)", tableAlias,
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index 0dc08a9..918eedf 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -23,8 +23,6 @@ import static com.google.common.base.Preconditions.checkArgument;
import java.io.Serializable;
import java.util.Locale;
-import java.util.function.Function;
-
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.metadata.datatype.DataType;
@@ -122,15 +120,6 @@ public class TblColRef implements Serializable {
private String identity;
private String parserDescription;
- /**
- * Function used to get quoted identitier
- */
- private transient Function<TblColRef, String> quotedFunc;
-
- public void setQuotedFunc(Function<TblColRef, String> quotedFunc) {
- this.quotedFunc = quotedFunc;
- }
-
TblColRef(ColumnDesc column) {
this.column = column;
}
@@ -249,13 +238,6 @@ public class TblColRef implements Serializable {
return identity;
}
- public String getQuotedIdentity() {
- if (quotedFunc == null)
- return getIdentity();
- else
- return quotedFunc.apply(this);
- }
-
@Override
public String toString() {
if (isInnerColumn() && parserDescription != null)
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
index 438fb4a..b536e29 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java
@@ -53,12 +53,12 @@ public class DefaultPartitionConditionBuilderTest extends LocalFileMetadataTestC
partitionDesc.setPartitionDateColumn(col.getCanonicalName());
partitionDesc.setPartitionDateFormat("yyyy-MM-dd");
TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23"));
- String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range, null);
+ String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'",
condition);
range = new TSRange(0L, 0L);
- condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range, null);
+ condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals("1=0", condition);
}
@@ -71,7 +71,7 @@ public class DefaultPartitionConditionBuilderTest extends LocalFileMetadataTestC
partitionDesc.setPartitionTimeFormat("HH");
TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"),
DateFormat.stringToMillis("2016-02-23 01:00:00"));
- String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range, null);
+ String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00' AND UNKNOWN_ALIAS.HOUR_COLUMN < '01'", condition);
}
@@ -88,7 +88,7 @@ public class DefaultPartitionConditionBuilderTest extends LocalFileMetadataTestC
partitionDesc.setPartitionTimeFormat("H");
TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"),
DateFormat.stringToMillis("2016-02-23 01:00:00"));
- String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range, null);
+ String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range);
Assert.assertEquals(
"((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN > '2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'))",
condition);
diff --git a/pom.xml b/pom.xml
index 3c691d8..84253be 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1640,9 +1640,6 @@
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
- <!-- Used to print file with unapproved licenses in project to stand output -->
- <consoleOutput>true</consoleOutput>
-
<!-- Exclude files/folders for apache release -->
<excludes>
<exclude>DEPENDENCIES</exclude>
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactory.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcDialect.java
similarity index 57%
copy from source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactory.java
copy to source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcDialect.java
index 498bc09..7e5ecee 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactory.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcDialect.java
@@ -15,24 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.kylin.source.jdbc.metadata;
+package org.apache.kylin.source.jdbc;
-import org.apache.kylin.common.SourceDialect;
-import org.apache.kylin.source.hive.DBConnConf;
-
-public class JdbcMetadataFactory {
-
- private JdbcMetadataFactory() {
- }
-
- public static IJdbcMetadata getJdbcMetadata(SourceDialect jdbcDialect, final DBConnConf dbConnConf) {
- switch (jdbcDialect) {
- case SQL_SERVER:
- return new SQLServerJdbcMetadata(dbConnConf);
- case MYSQL:
- return new MySQLJdbcMetadata(dbConnConf);
- default:
- return new DefaultJdbcMetadata(dbConnConf);
- }
- }
+public class JdbcDialect {
+ public static final String DIALECT_VERTICA = "vertica";
+ public static final String DIALECT_ORACLE = "oracle";
+ public static final String DIALECT_MYSQL = "mysql";
+ public static final String DIALECT_HIVE = "hive";
+ public static final String DIALECT_MSSQL = "mssql";
}
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index d728dcf..7eb4fa9 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -30,7 +30,6 @@ import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.SourceDialect;
import org.apache.kylin.common.util.DBUtils;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
@@ -51,7 +50,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
private static final Logger logger = LoggerFactory.getLogger(JdbcExplorer.class);
private final KylinConfig config;
- private final SourceDialect dialect;
+ private final String dialect;
private final DBConnConf dbconf;
private final IJdbcMetadata jdbcMetadataDialect;
@@ -62,7 +61,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
String jdbcUser = config.getJdbcSourceUser();
String jdbcPass = config.getJdbcSourcePass();
this.dbconf = new DBConnConf(driverClass, connectionUrl, jdbcUser, jdbcPass);
- this.dialect = SourceDialect.getDialect(config.getJdbcSourceDialect());
+ this.dialect = config.getJdbcSourceDialect();
this.jdbcMetadataDialect = JdbcMetadataFactory.getJdbcMetadata(dialect, dbconf);
}
@@ -118,7 +117,7 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
}
private String getSqlDataType(String javaDataType) {
- if (SourceDialect.VERTICA.equals(dialect) || SourceDialect.MYSQL.equals(dialect)) {
+ if (JdbcDialect.DIALECT_VERTICA.equals(dialect) || JdbcDialect.DIALECT_MSSQL.equals(dialect)) {
if (javaDataType.toLowerCase(Locale.ROOT).equals("double")) {
return "float";
}
@@ -133,9 +132,9 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
}
private String generateCreateSchemaSql(String schemaName) {
- if (SourceDialect.VERTICA.equals(dialect) || SourceDialect.MYSQL.equals(dialect)) {
+ if (JdbcDialect.DIALECT_VERTICA.equals(dialect) || JdbcDialect.DIALECT_MYSQL.equals(dialect)) {
return String.format(Locale.ROOT, "CREATE schema IF NOT EXISTS %s", schemaName);
- } else if (SourceDialect.SQL_SERVER.equals(dialect)) {
+ } else if (JdbcDialect.DIALECT_MSSQL.equals(dialect)) {
return String.format(Locale.ROOT,
"IF NOT EXISTS (SELECT name FROM sys.schemas WHERE name = N'%s') EXEC('CREATE SCHEMA"
+ " [%s] AUTHORIZATION [dbo]')",
@@ -152,13 +151,13 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
}
private String generateLoadDataSql(String tableName, String tableFileDir) {
- if (SourceDialect.VERTICA.equals(dialect)) {
+ if (JdbcDialect.DIALECT_VERTICA.equals(dialect)) {
return String.format(Locale.ROOT, "copy %s from local '%s/%s.csv' delimiter as ',';", tableName,
tableFileDir, tableName);
- } else if (SourceDialect.MYSQL.equals(dialect)) {
+ } else if (JdbcDialect.DIALECT_MYSQL.equals(dialect)) {
return String.format(Locale.ROOT, "LOAD DATA INFILE '%s/%s.csv' INTO %s FIELDS TERMINATED BY ',';",
tableFileDir, tableName, tableName);
- } else if (SourceDialect.SQL_SERVER.equals(dialect)) {
+ } else if (JdbcDialect.DIALECT_MSSQL.equals(dialect)) {
return String.format(Locale.ROOT, "BULK INSERT %s FROM '%s/%s.csv' WITH(FIELDTERMINATOR = ',')", tableName,
tableFileDir, tableName);
} else {
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java
index 94594f3..20f2dcb 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveInputBase.java
@@ -18,43 +18,28 @@
package org.apache.kylin.source.jdbc;
import com.google.common.collect.Maps;
-import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.SourceDialect;
import org.apache.kylin.common.util.SourceConfigurationUtil;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
import org.apache.kylin.job.JoinedFlatTable;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
+import org.apache.kylin.job.util.FlatTableSqlQuoteUtils;
import org.apache.kylin.metadata.TableMetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.hive.DBConnConf;
import org.apache.kylin.source.hive.HiveInputBase;
-import org.apache.kylin.source.jdbc.metadata.IJdbcMetadata;
-import org.apache.kylin.source.jdbc.metadata.JdbcMetadataFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.ResultSet;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
public class JdbcHiveInputBase extends HiveInputBase {
private static final Logger logger = LoggerFactory.getLogger(JdbcHiveInputBase.class);
@@ -62,66 +47,9 @@ public class JdbcHiveInputBase extends HiveInputBase {
private static final String DEFAULT_QUEUE = "default";
public static class JdbcBaseBatchCubingInputSide extends BaseBatchCubingInputSide {
- private IJdbcMetadata jdbcMetadataDialect;
- private DBConnConf dbconf;
- private SourceDialect dialect;
- private final Map<String, String> metaMap = new TreeMap<>();
public JdbcBaseBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
super(flatDesc);
- KylinConfig config = KylinConfig.getInstanceFromEnv();
- String connectionUrl = config.getJdbcSourceConnectionUrl();
- String driverClass = config.getJdbcSourceDriver();
- String jdbcUser = config.getJdbcSourceUser();
- String jdbcPass = config.getJdbcSourcePass();
- dbconf = new DBConnConf(driverClass, connectionUrl, jdbcUser, jdbcPass);
- dialect = SourceDialect.getDialect(config.getJdbcSourceDialect());
- jdbcMetadataDialect = JdbcMetadataFactory.getJdbcMetadata(dialect, dbconf);
- calCachedJdbcMeta(metaMap, dbconf, jdbcMetadataDialect);
- if (logger.isTraceEnabled()) {
- StringBuilder dumpInfo = new StringBuilder();
- metaMap.forEach((k, v) -> dumpInfo.append("CachedMetadata: ").append(k).append(" => ").append(v)
- .append(System.lineSeparator()));
- logger.trace(dumpInfo.toString());
- }
- }
-
- /**
- * Fetch and cache metadata from JDBC API, which will help to resolve
- * case-sensitive problem of sql identifier
- *
- * @param metadataMap a Map which mapping upper case identifier to real/original identifier
- */
- public static void calCachedJdbcMeta(Map<String, String> metadataMap, DBConnConf dbconf,
- IJdbcMetadata jdbcMetadataDialect) {
- try (Connection connection = SqlUtil.getConnection(dbconf)) {
- DatabaseMetaData databaseMetaData = connection.getMetaData();
- for (String database : jdbcMetadataDialect.listDatabases()) {
- metadataMap.put(database.toUpperCase(Locale.ROOT), database);
- ResultSet tableRs = jdbcMetadataDialect.getTable(databaseMetaData, database, null);
- while (tableRs.next()) {
- String tableName = tableRs.getString("TABLE_NAME");
- ResultSet colRs = jdbcMetadataDialect.listColumns(databaseMetaData, database, tableName);
- while (colRs.next()) {
- String colName = colRs.getString("COLUMN_NAME");
- colName = database + "." + tableName + "." + colName;
- metadataMap.put(colName.toUpperCase(Locale.ROOT), colName);
- }
- colRs.close();
- tableName = database + "." + tableName;
- metadataMap.put(tableName.toUpperCase(Locale.ROOT), tableName);
- }
- tableRs.close();
- }
- } catch (IllegalStateException e) {
- if (SqlUtil.DRIVER_MISS.equalsIgnoreCase(e.getMessage())) {
- logger.warn("Ignore JDBC Driver Missing in yarn node.", e);
- } else {
- throw e;
- }
- } catch (Exception e) {
- throw new IllegalStateException("Error when connect to JDBC source " + dbconf.getUrl(), e);
- }
}
protected KylinConfig getConfig() {
@@ -220,19 +148,22 @@ public class JdbcHiveInputBase extends HiveInputBase {
partCol = partitionDesc.getPartitionDateColumn();//tablename.colname
}
+ String splitTable;
String splitTableAlias;
String splitColumn;
String splitDatabase;
TblColRef splitColRef = determineSplitColumn();
+ splitTable = splitColRef.getTableRef().getTableName();
splitTableAlias = splitColRef.getTableAlias();
-
- splitColumn = getColumnIdentityQuoted(splitColRef, jdbcMetadataDialect, metaMap, true);
+ splitColumn = JoinedFlatTable.getQuotedColExpressionInSourceDB(flatDesc, splitColRef);
splitDatabase = splitColRef.getColumnDesc().getTable().getDatabase();
- String selectSql = generateSelectDataStatementRDBMS(flatDesc, true, new String[] { partCol },
- jdbcMetadataDialect, metaMap);
+ //using sqoop to extract data from jdbc source and dump them to hive
+ String selectSql = JoinedFlatTable.generateSelectDataStatement(flatDesc, true, new String[] { partCol });
selectSql = escapeQuotationInSql(selectSql);
+
+
String hiveTable = flatDesc.getTableName();
String connectionUrl = config.getJdbcSourceConnectionUrl();
String driverClass = config.getJdbcSourceDriver();
@@ -244,19 +175,17 @@ public class JdbcHiveInputBase extends HiveInputBase {
String filedDelimiter = config.getJdbcSourceFieldDelimiter();
int mapperNum = config.getSqoopMapperNum();
- String bquery = String.format(Locale.ROOT, "SELECT min(%s), max(%s) FROM %s.%s ", splitColumn, splitColumn,
- getSchemaQuoted(metaMap, splitDatabase, jdbcMetadataDialect, true),
- getTableIdentityQuoted(splitColRef.getTableRef(), metaMap, jdbcMetadataDialect, true));
+ String bquery = String.format(Locale.ROOT, "SELECT min(%s), max(%s) FROM %s.%s as %s", splitColumn,
+ splitColumn, splitDatabase, splitTable, splitTableAlias);
if (partitionDesc.isPartitioned()) {
SegmentRange segRange = flatDesc.getSegRange();
if (segRange != null && !segRange.isInfinite()) {
if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias)
&& (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc
- .getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
-
- String quotedPartCond = partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(
- partitionDesc, flatDesc.getSegment(), segRange,
- col -> getTableColumnIdentityQuoted(col, jdbcMetadataDialect, metaMap, true));
+ .getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
+ String quotedPartCond = FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr(flatDesc,
+ partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc,
+ flatDesc.getSegment(), segRange));
bquery += " WHERE " + quotedPartCond;
}
}
@@ -266,13 +195,14 @@ public class JdbcHiveInputBase extends HiveInputBase {
// escape ` in cmd
splitColumn = escapeQuotationInSql(splitColumn);
- String cmd = String.format(Locale.ROOT, "%s/bin/sqoop import" + generateSqoopConfigArgString()
- + "--connect \"%s\" --driver %s --username %s --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
- + "--target-dir %s/%s --split-by %s --boundary-query \"%s\" --null-string '%s' "
- + "--null-non-string '%s' --fields-terminated-by '%s' --num-mappers %d", sqoopHome, connectionUrl,
- driverClass, jdbcUser, jdbcPass, selectSql, jobWorkingDir, hiveTable, splitColumn, bquery,
- sqoopNullString, sqoopNullNonString, filedDelimiter, mapperNum);
- logger.debug("sqoop cmd : {}", cmd);
+ String cmd = String.format(Locale.ROOT,
+ "%s/bin/sqoop import" + generateSqoopConfigArgString()
+ + "--connect \"%s\" --driver %s --username %s --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
+ + "--target-dir %s/%s --split-by %s --boundary-query \"%s\" --null-string '%s' "
+ + "--null-non-string '%s' --fields-terminated-by '%s' --num-mappers %d",
+ sqoopHome, connectionUrl, driverClass, jdbcUser, jdbcPass, selectSql, jobWorkingDir, hiveTable,
+ splitColumn, bquery, sqoopNullString, sqoopNullNonString, filedDelimiter, mapperNum);
+ logger.debug(String.format(Locale.ROOT, "sqoop cmd:%s", cmd));
CmdStep step = new CmdStep();
step.setCmd(cmd);
step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE);
@@ -282,7 +212,7 @@ public class JdbcHiveInputBase extends HiveInputBase {
protected String generateSqoopConfigArgString() {
KylinConfig kylinConfig = getConfig();
Map<String, String> config = Maps.newHashMap();
- config.put(MR_OVERRIDE_QUEUE_KEY, getSqoopJobQueueName(kylinConfig)); // override job queue from mapreduce config
+ config.put("mapreduce.job.queuename", getSqoopJobQueueName(kylinConfig)); // override job queue from mapreduce config
config.putAll(SourceConfigurationUtil.loadSqoopConfiguration());
config.putAll(kylinConfig.getSqoopConfigOverride());
@@ -299,232 +229,4 @@ public class JdbcHiveInputBase extends HiveInputBase {
sqlExpr = sqlExpr.replaceAll("`", "\\\\`");
return sqlExpr;
}
-
- private static String generateSelectDataStatementRDBMS(IJoinedFlatTableDesc flatDesc, boolean singleLine,
- String[] skipAs, IJdbcMetadata metadata, Map<String, String> metaMap) {
- SourceDialect dialect = metadata.getDialect();
- final String sep = singleLine ? " " : "\n";
-
- final List<String> skipAsList = (skipAs == null) ? new ArrayList<>() : Arrays.asList(skipAs);
-
- StringBuilder sql = new StringBuilder();
- sql.append("SELECT");
- sql.append(sep);
-
- for (int i = 0; i < flatDesc.getAllColumns().size(); i++) {
- TblColRef col = flatDesc.getAllColumns().get(i);
- if (i > 0) {
- sql.append(",");
- }
- String colTotalName = String.format(Locale.ROOT, "%s.%s", col.getTableRef().getTableName(), col.getName());
- if (skipAsList.contains(colTotalName)) {
- sql.append(getTableColumnIdentityQuoted(col, metadata, metaMap, true)).append(sep);
- } else {
- sql.append(getTableColumnIdentityQuoted(col, metadata, metaMap, true)).append(" as ")
- .append(quoteIdentifier(JoinedFlatTable.colName(col), dialect)).append(sep);
- }
- }
- appendJoinStatement(flatDesc, sql, singleLine, metadata, metaMap);
- appendWhereStatement(flatDesc, sql, singleLine, metadata, metaMap);
- return sql.toString();
- }
-
- private static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
- IJdbcMetadata metadata, Map<String, String> metaMap) {
- final String sep = singleLine ? " " : "\n";
- Set<TableRef> dimTableCache = new HashSet<>();
-
- DataModelDesc model = flatDesc.getDataModel();
- sql.append(" FROM ")
- .append(getSchemaQuoted(metaMap,
- flatDesc.getDataModel().getRootFactTable().getTableDesc().getDatabase(), metadata, true))
- .append(".")
- .append(getTableIdentityQuoted(flatDesc.getDataModel().getRootFactTable(), metaMap, metadata, true));
-
- sql.append(" ");
- sql.append((getTableIdentityQuoted(flatDesc.getDataModel().getRootFactTable(), metaMap, metadata, true)))
- .append(sep);
-
- for (JoinTableDesc lookupDesc : model.getJoinTables()) {
- JoinDesc join = lookupDesc.getJoin();
- if (join != null && !join.getType().equals("")) {
- TableRef dimTable = lookupDesc.getTableRef();
- if (!dimTableCache.contains(dimTable)) {
- TblColRef[] pk = join.getPrimaryKeyColumns();
- TblColRef[] fk = join.getForeignKeyColumns();
- if (pk.length != fk.length) {
- throw new RuntimeException("Invalid join condition of lookup table:" + lookupDesc);
- }
- String joinType = join.getType().toUpperCase(Locale.ROOT);
-
- sql.append(joinType).append(" JOIN ")
- .append(getSchemaQuoted(metaMap, dimTable.getTableDesc().getDatabase(), metadata, true))
- .append(".").append(getTableIdentityQuoted(dimTable, metaMap, metadata, true));
-
- sql.append(" ");
- sql.append(getTableIdentityQuoted(dimTable, metaMap, metadata, true)).append(sep);
- sql.append("ON ");
- for (int i = 0; i < pk.length; i++) {
- if (i > 0) {
- sql.append(" AND ");
- }
- sql.append(getTableColumnIdentityQuoted(fk[i], metadata, metaMap, true)).append(" = ")
- .append(getTableColumnIdentityQuoted(pk[i], metadata, metaMap, true));
- }
- sql.append(sep);
- dimTableCache.add(dimTable);
- }
- }
- }
- }
-
- private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
- IJdbcMetadata metadata, Map<String, String> metaMap) {
- final String sep = singleLine ? " " : "\n";
-
- StringBuilder whereBuilder = new StringBuilder();
- whereBuilder.append("WHERE 1=1");
-
- DataModelDesc model = flatDesc.getDataModel();
- if (StringUtils.isNotEmpty(model.getFilterCondition())) {
- whereBuilder.append(" AND (").append(model.getFilterCondition()).append(") ");
- }
-
- if (flatDesc.getSegment() != null) {
- PartitionDesc partDesc = model.getPartitionDesc();
- if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
- SegmentRange segRange = flatDesc.getSegRange();
-
- if (segRange != null && !segRange.isInfinite()) {
- whereBuilder.append(" AND (");
- whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc,
- flatDesc.getSegment(), segRange,
- col -> getTableColumnIdentityQuoted(col, metadata, metaMap, true)));
- whereBuilder.append(")");
- whereBuilder.append(sep);
- }
- }
- }
- sql.append(whereBuilder.toString());
- }
-
- /**
- * @return {TABLE_NAME}.{COLUMN_NAME}
- */
- private static String getTableColumnIdentityQuoted(TblColRef col, IJdbcMetadata metadata,
- Map<String, String> metaMap, boolean needQuote) {
- String tblName = getTableIdentityQuoted(col.getTableRef(), metaMap, metadata, needQuote);
- String colName = getColumnIdentityQuoted(col, metadata, metaMap, needQuote);
- return tblName + "." + colName;
- }
-
- /**
- * @return {SCHEMA_NAME}
- */
- static String getSchemaQuoted(Map<String, String> metaMap, String database, IJdbcMetadata metadata,
- boolean needQuote) {
- String databaseName = fetchValue(database, null, null, metaMap);
- if (needQuote) {
- return quoteIdentifier(databaseName, metadata.getDialect());
- } else {
- return databaseName;
- }
- }
-
- /**
- * @return {TABLE_NAME}
- */
- static String getTableIdentityQuoted(TableRef tableRef, Map<String, String> metaMap, IJdbcMetadata metadata,
- boolean needQuote) {
- String value = fetchValue(tableRef.getTableDesc().getDatabase(), tableRef.getTableDesc().getName(), null,
- metaMap);
- String[] res = value.split("\\.");
- value = res[res.length - 1];
- if (needQuote) {
- return quoteIdentifier(value, metadata.getDialect());
- } else {
- return value;
- }
- }
-
- /**
- * @return {TABLE_NAME}
- */
- static String getTableIdentityQuoted(String database, String table, Map<String, String> metaMap,
- IJdbcMetadata metadata, boolean needQuote) {
- String value = fetchValue(database, table, null, metaMap);
- String[] res = value.split("\\.");
- value = res[res.length - 1];
- if (needQuote) {
- return quoteIdentifier(value, metadata.getDialect());
- } else {
- return value;
- }
- }
-
- /**
- * @return {COLUMN_NAME}
- */
- private static String getColumnIdentityQuoted(TblColRef tblColRef, IJdbcMetadata metadata,
- Map<String, String> metaMap, boolean needQuote) {
- String value = fetchValue(tblColRef.getTableRef().getTableDesc().getDatabase(),
- tblColRef.getTableRef().getTableDesc().getName(), tblColRef.getName(), metaMap);
- String[] res = value.split("\\.");
- value = res[res.length - 1];
- if (needQuote) {
- return quoteIdentifier(value, metadata.getDialect());
- } else {
- return value;
- }
- }
-
- /**
- * Quote the identifier acccording to sql dialect, as far as I know,
- * MySQL use backtick(`), oracle 11g use double quotation("), sql server 2017
- * use square brackets([ or ]) as quote character.
- *
- * @param identifier something looks like tableA.columnB
- */
- static String quoteIdentifier(String identifier, SourceDialect dialect) {
- if (KylinConfig.getInstanceFromEnv().enableHiveDdlQuote()) {
- String[] identifierArray = identifier.split("\\.");
- String quoted = "";
- for (int i = 0; i < identifierArray.length; i++) {
- switch (dialect) {
- case SQL_SERVER:
- identifierArray[i] = "[" + identifierArray[i] + "]";
- break;
- case MYSQL:
- case HIVE:
- identifierArray[i] = "`" + identifierArray[i] + "`";
- break;
- default:
- String quote = KylinConfig.getInstanceFromEnv().getQuoteCharacter();
- identifierArray[i] = quote + identifierArray[i] + quote;
- }
- }
- quoted = String.join(".", identifierArray);
- return quoted;
- } else {
- return identifier;
- }
- }
-
- static String fetchValue(String database, String table, String column, Map<String, String> metadataMap) {
- String key;
- if (table == null && column == null) {
- key = database;
- } else if (column == null) {
- key = database + "." + table;
- } else {
- key = database + "." + table + "." + column;
- }
- String val = metadataMap.get(key.toUpperCase(Locale.ROOT));
- if (val == null) {
- logger.warn("Not find for {} from metadata cache.", key);
- return key;
- } else {
- return val;
- }
- }
}
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcTableReader.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcTableReader.java
index 1c689dd..3c2b4f9 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcTableReader.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/JdbcTableReader.java
@@ -23,15 +23,10 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Locale;
-import java.util.Map;
-import java.util.TreeMap;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.SourceDialect;
import org.apache.kylin.source.IReadableTable.TableReader;
import org.apache.kylin.source.hive.DBConnConf;
-import org.apache.kylin.source.jdbc.metadata.IJdbcMetadata;
-import org.apache.kylin.source.jdbc.metadata.JdbcMetadataFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,15 +61,7 @@ public class JdbcTableReader implements TableReader {
String jdbcPass = config.getJdbcSourcePass();
dbconf = new DBConnConf(driverClass, connectionUrl, jdbcUser, jdbcPass);
jdbcCon = SqlUtil.getConnection(dbconf);
- IJdbcMetadata meta = JdbcMetadataFactory
- .getJdbcMetadata(SourceDialect.getDialect(config.getJdbcSourceDialect()), dbconf);
-
- Map<String, String> metadataCache = new TreeMap<>();
- JdbcHiveInputBase.JdbcBaseBatchCubingInputSide.calCachedJdbcMeta(metadataCache, dbconf, meta);
- String database = JdbcHiveInputBase.getSchemaQuoted(metadataCache, dbName, meta, true);
- String table = JdbcHiveInputBase.getTableIdentityQuoted(dbName, tableName, metadataCache, meta, true);
-
- String sql = String.format(Locale.ROOT, "select * from %s.%s", database, table);
+ String sql = String.format(Locale.ROOT, "select * from %s.%s", dbName, tableName);
try {
statement = jdbcCon.createStatement();
rs = statement.executeQuery(sql);
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
index 9299d78..5242832 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/SqlUtil.java
@@ -23,6 +23,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.util.Random;
+
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.source.hive.DBConnConf;
import org.slf4j.Logger;
@@ -61,7 +62,6 @@ public class SqlUtil {
}
public static final int tryTimes = 5;
- public static final String DRIVER_MISS = "DRIVER_MISS";
public static Connection getConnection(DBConnConf dbconf) {
if (dbconf.getUrl() == null)
@@ -70,8 +70,7 @@ public class SqlUtil {
try {
Class.forName(dbconf.getDriver());
} catch (Exception e) {
- logger.error("Miss Driver", e);
- throw new IllegalStateException(DRIVER_MISS);
+ logger.error("", e);
}
boolean got = false;
int times = 0;
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveInputBase.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveInputBase.java
index fcafae2..9fd6d30 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveInputBase.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/extensible/JdbcHiveInputBase.java
@@ -94,7 +94,7 @@ public class JdbcHiveInputBase extends org.apache.kylin.source.jdbc.JdbcHiveInpu
.getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
String quotedPartCond = FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr(flatDesc,
partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc,
- flatDesc.getSegment(), segRange, null));
+ flatDesc.getSegment(), segRange));
bquery += " WHERE " + quotedPartCond;
}
}
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/DefaultJdbcMetadata.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/DefaultJdbcMetadata.java
index b9c65fc..0842199 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/DefaultJdbcMetadata.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/DefaultJdbcMetadata.java
@@ -23,8 +23,8 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
+
import java.util.Locale;
-import org.apache.kylin.common.SourceDialect;
import org.apache.kylin.source.hive.DBConnConf;
import org.apache.kylin.source.jdbc.SqlUtil;
import org.slf4j.Logger;
@@ -74,8 +74,4 @@ public class DefaultJdbcMetadata implements IJdbcMetadata {
public ResultSet listColumns(final DatabaseMetaData dbmd, String schema, String table) throws SQLException {
return dbmd.getColumns(null, schema, table, null);
}
-
- public SourceDialect getDialect() {
- return SourceDialect.UNKNOWN;
- }
}
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/IJdbcMetadata.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/IJdbcMetadata.java
index f41c3e8..169fe60 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/IJdbcMetadata.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/IJdbcMetadata.java
@@ -17,16 +17,12 @@
*/
package org.apache.kylin.source.jdbc.metadata;
-import org.apache.kylin.common.SourceDialect;
-
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
public interface IJdbcMetadata {
- SourceDialect getDialect();
-
List<String> listDatabases() throws SQLException;
List<String> listTables(String database) throws SQLException;
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactory.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactory.java
index 498bc09..ae4c0ff 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactory.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactory.java
@@ -17,19 +17,17 @@
*/
package org.apache.kylin.source.jdbc.metadata;
-import org.apache.kylin.common.SourceDialect;
+import java.util.Locale;
import org.apache.kylin.source.hive.DBConnConf;
+import org.apache.kylin.source.jdbc.JdbcDialect;
-public class JdbcMetadataFactory {
-
- private JdbcMetadataFactory() {
- }
-
- public static IJdbcMetadata getJdbcMetadata(SourceDialect jdbcDialect, final DBConnConf dbConnConf) {
+public abstract class JdbcMetadataFactory {
+ public static IJdbcMetadata getJdbcMetadata(String dialect, final DBConnConf dbConnConf) {
+ String jdbcDialect = (null == dialect) ? "" : dialect.toLowerCase(Locale.ROOT);
switch (jdbcDialect) {
- case SQL_SERVER:
+ case (JdbcDialect.DIALECT_MSSQL):
return new SQLServerJdbcMetadata(dbConnConf);
- case MYSQL:
+ case (JdbcDialect.DIALECT_MYSQL):
return new MySQLJdbcMetadata(dbConnConf);
default:
return new DefaultJdbcMetadata(dbConnConf);
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java
index e3c523c..54c2a03 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/MySQLJdbcMetadata.java
@@ -24,7 +24,6 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.kylin.common.SourceDialect;
import org.apache.kylin.source.hive.DBConnConf;
import org.apache.kylin.source.jdbc.SqlUtil;
@@ -65,9 +64,4 @@ public class MySQLJdbcMetadata extends DefaultJdbcMetadata {
public ResultSet getTable(final DatabaseMetaData dbmd, String catalog, String table) throws SQLException {
return dbmd.getTables(catalog, null, table, null);
}
-
- @Override
- public SourceDialect getDialect() {
- return SourceDialect.MYSQL;
- }
}
diff --git a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadata.java b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadata.java
index 696a350..5373672 100644
--- a/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadata.java
+++ b/source-jdbc/src/main/java/org/apache/kylin/source/jdbc/metadata/SQLServerJdbcMetadata.java
@@ -26,7 +26,6 @@ import java.util.List;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.SourceDialect;
import org.apache.kylin.source.hive.DBConnConf;
import org.apache.kylin.source.jdbc.SqlUtil;
@@ -60,9 +59,4 @@ public class SQLServerJdbcMetadata extends DefaultJdbcMetadata {
}
return new ArrayList<>(ret);
}
-
- @Override
- public SourceDialect getDialect() {
- return SourceDialect.SQL_SERVER;
- }
}
diff --git a/source-jdbc/src/test/java/org/apache/kylin/source/jdbc/JdbcExplorerTest.java b/source-jdbc/src/test/java/org/apache/kylin/source/jdbc/JdbcExplorerTest.java
index ed3d181..a0df4f4 100644
--- a/source-jdbc/src/test/java/org/apache/kylin/source/jdbc/JdbcExplorerTest.java
+++ b/source-jdbc/src/test/java/org/apache/kylin/source/jdbc/JdbcExplorerTest.java
@@ -18,6 +18,7 @@
package org.apache.kylin.source.jdbc;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
@@ -34,7 +35,6 @@ import java.util.List;
import java.util.Locale;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.SourceDialect;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.model.ColumnDesc;
@@ -83,7 +83,7 @@ public class JdbcExplorerTest extends LocalFileMetadataTestCase {
PowerMockito.stub(PowerMockito.method(SqlUtil.class, "getConnection")).toReturn(connection);
PowerMockito.mockStatic(JdbcMetadataFactory.class);
- when(JdbcMetadataFactory.getJdbcMetadata(any(SourceDialect.class), any(DBConnConf.class))).thenReturn(jdbcMetadata);
+ when(JdbcMetadataFactory.getJdbcMetadata(anyString(), any(DBConnConf.class))).thenReturn(jdbcMetadata);
when(connection.getMetaData()).thenReturn(dbmd);
jdbcExplorer = spy(JdbcExplorer.class);
diff --git a/source-jdbc/src/test/java/org/apache/kylin/source/jdbc/JdbcHiveInputBaseTest.java b/source-jdbc/src/test/java/org/apache/kylin/source/jdbc/JdbcHiveInputBaseTest.java
deleted file mode 100644
index f6415e6..0000000
--- a/source-jdbc/src/test/java/org/apache/kylin/source/jdbc/JdbcHiveInputBaseTest.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.source.jdbc;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.SourceDialect;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-public class JdbcHiveInputBaseTest extends LocalFileMetadataTestCase {
-
- @BeforeClass
- public static void setupClass() throws SQLException {
- staticCreateTestMetadata();
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- kylinConfig.setProperty("kylin.source.hive.quote-enabled", "true");
- }
-
- @Test
- public void testFetchValue() {
- Map<String, String> map = new HashMap<>();
- String guess = JdbcHiveInputBase.fetchValue("DB_1", "TB_2", "COL_3", map);
-
- // not found, return input value
- assertEquals("DB_1.TB_2.COL_3", guess);
- map.put("DB_1.TB_2.COL_3", "Db_1.Tb_2.Col_3");
-
- guess = JdbcHiveInputBase.fetchValue("DB_1", "TB_2", "COL_3", map);
- // found, return cached value
- assertEquals("Db_1.Tb_2.Col_3", guess);
- }
-
- @Test
- public void testQuoteIdentifier() {
- String guess = JdbcHiveInputBase.quoteIdentifier("Tbl1.Col1", SourceDialect.MYSQL);
- assertEquals("`Tbl1`.`Col1`", guess);
- guess = JdbcHiveInputBase.quoteIdentifier("Tbl1.Col1", SourceDialect.SQL_SERVER);
- assertEquals("[Tbl1].[Col1]", guess);
- }
-
- @AfterClass
- public static void clenup() {
- staticCleanupTestMetadata();
- }
-}
diff --git a/core-common/src/main/java/org/apache/kylin/common/SourceDialect.java b/source-jdbc/src/test/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactoryTest.java
similarity index 52%
rename from core-common/src/main/java/org/apache/kylin/common/SourceDialect.java
rename to source-jdbc/src/test/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactoryTest.java
index a87054d..d9c7425 100644
--- a/core-common/src/main/java/org/apache/kylin/common/SourceDialect.java
+++ b/source-jdbc/src/test/java/org/apache/kylin/source/jdbc/metadata/JdbcMetadataFactoryTest.java
@@ -15,45 +15,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.kylin.common;
-
-/**
- * Decide sql pattern according to dialect from differenct data source
- */
-public enum SourceDialect {
- HIVE("hive"),
-
- /**
- * Support MySQL 5.7
- */
- MYSQL("mysql"),
-
- /**
- * Support Microsoft Sql Server 2017
- */
- SQL_SERVER("mssql"),
-
- VERTICA("vertica"),
-
- /**
- * Others
- */
- UNKNOWN("unknown");
-
- String source;
-
- SourceDialect(String source) {
- this.source = source;
- }
-
- public static SourceDialect getDialect(String name) {
-
- for (SourceDialect dialect : SourceDialect.values()) {
- if (dialect.source.equalsIgnoreCase(name)) {
- return dialect;
- }
- }
- return UNKNOWN;
+package org.apache.kylin.source.jdbc.metadata;
+
+import org.apache.kylin.source.jdbc.JdbcDialect;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class JdbcMetadataFactoryTest {
+
+ @Test
+ public void testGetJdbcMetadata() {
+ Assert.assertTrue(
+ JdbcMetadataFactory.getJdbcMetadata(JdbcDialect.DIALECT_MSSQL, null) instanceof SQLServerJdbcMetadata);
+ Assert.assertTrue(
+ JdbcMetadataFactory.getJdbcMetadata(JdbcDialect.DIALECT_MYSQL, null) instanceof MySQLJdbcMetadata);
+ Assert.assertTrue(
+ JdbcMetadataFactory.getJdbcMetadata(JdbcDialect.DIALECT_VERTICA, null) instanceof DefaultJdbcMetadata);
}
}
[kylin] 07/12: KYLIN-4055 cube quey and ad-hoc query return
different meta info
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 32ad065b56d3ff2b91a0de585b21480b19d1f853
Author: javalife0312 <ja...@126.com>
AuthorDate: Thu Sep 5 10:44:36 2019 +0800
KYLIN-4055 cube quey and ad-hoc query return different meta info
---
.../java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
index 315a3c1..0529a71 100644
--- a/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
+++ b/query/src/main/java/org/apache/kylin/query/adhoc/PushDownRunnerJdbcImpl.java
@@ -132,7 +132,7 @@ public class PushDownRunnerJdbcImpl extends AbstractPushdownRunner {
columnLabel = resultSetMetaData.getColumnLabel(columnIndex);
// Suppose column label has format [table name].[column name]
if (columnLabel.contains(".")) {
- columnLabel = StringUtils.substringAfterLast(columnLabel, ".");
+ columnLabel = StringUtils.substringAfterLast(columnLabel, ".").toUpperCase(Locale.ROOT);
}
} catch (SQLException e) {
// Fall back to default value
@@ -140,7 +140,7 @@ public class PushDownRunnerJdbcImpl extends AbstractPushdownRunner {
String columnName = null;
try {
- columnName = resultSetMetaData.getColumnName(columnIndex);
+ columnName = resultSetMetaData.getColumnName(columnIndex).toUpperCase(Locale.ROOT);
} catch (SQLException e) {
// Fall back to default value
}
[kylin] 03/12: KYLIN-4131 Fix Broadcaster memory leak
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8bd6bedbc26da9d6acd9bf2232688bfaf8804e92
Author: chao long <wa...@qq.com>
AuthorDate: Mon Aug 12 23:15:11 2019 +0800
KYLIN-4131 Fix Broadcaster memory leak
---
.../java/org/apache/kylin/common/Closeable.java | 23 ++++++++++++++++++++++
.../java/org/apache/kylin/common/KylinConfig.java | 15 ++++++++++++++
.../kylin/metadata/cachesync/Broadcaster.java | 12 ++++++++---
3 files changed, 47 insertions(+), 3 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/Closeable.java b/core-common/src/main/java/org/apache/kylin/common/Closeable.java
new file mode 100644
index 0000000..e5e3de6
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/Closeable.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common;
+
+public interface Closeable {
+ void close();
+}
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 8951342..862be54 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -475,7 +475,22 @@ public class KylinConfig extends KylinConfigBase {
return;
}
+ Map<Class, Closeable> closableManagers = new ConcurrentHashMap<>();
+
+ managersCache.forEach((key, value) -> {
+ if (value instanceof Closeable) {
+ closableManagers.put(key, (Closeable) value);
+ }
+ });
+
managersCache.clear();
+
+ if (closableManagers.size() > 0) {
+ closableManagers.forEach((key, value) -> {
+ logger.info("Close manager {}", key.getSimpleName());
+ value.close();
+ });
+ }
}
public Properties exportToProperties() {
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
index 834401c..7e6ee7c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/cachesync/Broadcaster.java
@@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.Closeable;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.restclient.RestClient;
import org.apache.kylin.common.util.ClassUtil;
@@ -57,7 +58,7 @@ import com.google.common.collect.Maps;
* - on all servers, model listener is invoked, reload the model, and notify a "project_schema" update event
* - all listeners respond to the "project_schema" update -- reload cube desc, clear project L2 cache, clear calcite data source etc
*/
-public class Broadcaster {
+public class Broadcaster implements Closeable {
private static final Logger logger = LoggerFactory.getLogger(Broadcaster.class);
@@ -153,6 +154,11 @@ public class Broadcaster {
});
}
+ @Override
+ public void close() {
+ stopAnnounce();
+ }
+
private SyncErrorHandler getSyncErrorHandler(KylinConfig config) {
String clzName = config.getCacheSyncErrorHandler();
if (StringUtils.isEmpty(clzName)) {
@@ -166,8 +172,8 @@ public class Broadcaster {
}
public void stopAnnounce() {
- announceThreadPool.shutdown();
- announceMainLoop.shutdown();
+ announceThreadPool.shutdownNow();
+ announceMainLoop.shutdownNow();
}
// static listener survives cache wipe and goes after normal listeners
[kylin] 12/12: KYLIN-3628 Fix regression bug
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c6cfbff3d17687a348f14d612aa5dd119f97ecd0
Author: XiaoxiangYu <hi...@126.com>
AuthorDate: Mon Sep 23 10:50:50 2019 +0800
KYLIN-3628 Fix regression bug
Previous code change will cause NPE for "SELECT * from DIMENSION_TABLE" when dimension table name use alias.
---
.../java/org/apache/kylin/cube/CubeManager.java | 36 ++++++++++++----------
1 file changed, 20 insertions(+), 16 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 6a96f18..95d01bc 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -1231,31 +1231,35 @@ public class CubeManager implements IRealizationProvider {
public CubeInstance findLatestSnapshot(List<RealizationEntry> realizationEntries, String lookupTableName,
CubeInstance cubeInstance) {
CubeInstance cube = null;
- if (!realizationEntries.isEmpty()) {
- long maxBuildTime = Long.MIN_VALUE;
- RealizationRegistry registry = RealizationRegistry.getInstance(config);
- for (RealizationEntry entry : realizationEntries) {
- IRealization realization = registry.getRealization(entry.getType(), entry.getRealization());
- if (realization != null && realization.isReady() && realization instanceof CubeInstance) {
- CubeInstance current = (CubeInstance) realization;
- if (checkMeetSnapshotTable(current, lookupTableName)) {
- CubeSegment segment = current.getLatestReadySegment();
- if (segment != null) {
- long latestBuildTime = segment.getLastBuildTime();
- if (latestBuildTime > maxBuildTime) {
- maxBuildTime = latestBuildTime;
- cube = current;
+ try {
+ if (!realizationEntries.isEmpty()) {
+ long maxBuildTime = Long.MIN_VALUE;
+ RealizationRegistry registry = RealizationRegistry.getInstance(config);
+ for (RealizationEntry entry : realizationEntries) {
+ IRealization realization = registry.getRealization(entry.getType(), entry.getRealization());
+ if (realization != null && realization.isReady() && realization instanceof CubeInstance) {
+ CubeInstance current = (CubeInstance) realization;
+ if (checkMeetSnapshotTable(current, lookupTableName)) {
+ CubeSegment segment = current.getLatestReadySegment();
+ if (segment != null) {
+ long latestBuildTime = segment.getLastBuildTime();
+ if (latestBuildTime > maxBuildTime) {
+ maxBuildTime = latestBuildTime;
+ cube = current;
+ }
}
}
}
}
}
+ } catch (Exception e) {
+ logger.info("Unexpected error.", e);
}
if (!cubeInstance.equals(cube)) {
logger.debug("Picked cube {} over {} as it provides a more recent snapshot of the lookup table {}", cube,
cubeInstance, lookupTableName);
}
- return cube;
+ return cube == null ? cubeInstance : cube;
}
/**
@@ -1270,7 +1274,7 @@ public class CubeManager implements IRealizationProvider {
lookupTbl = strArr[strArr.length - 1];
}
for (DimensionDesc dimensionDesc : toCheck.getDescriptor().getDimensions()) {
- if (dimensionDesc.getTable().equalsIgnoreCase(lookupTbl)) {
+ if (dimensionDesc.getTableRef().getTableName().equalsIgnoreCase(lookupTbl)) {
checkRes = true;
break;
}