You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2022/12/16 07:01:32 UTC
[kylin] 01/15: fix ts query pushdown after force ts
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d7269e764109a32664e48a56bfd9484192e1cf9c
Author: Shuai li <lo...@live.cn>
AuthorDate: Thu Oct 20 11:50:44 2022 +0800
fix ts query pushdown after force ts
Co-authored-by: shuai.li <sh...@kyligence.io>
---
.../engine/exec/sparder/SparderQueryPlanExec.java | 8 ++++
.../kap/secondstorage/SecondStorageLockTest.java | 50 ++++++++++++++++------
2 files changed, 44 insertions(+), 14 deletions(-)
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
index bbe7d25192..225c110baf 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
@@ -124,6 +124,7 @@ public class SparderQueryPlanExec implements QueryPlanExec {
}
QueryContext.current().setLastFailed(true);
cause = retryException;
+ checkOnlyTsAnswer();
}
}
if (forceTableIndexAtException(e)) {
@@ -207,4 +208,11 @@ public class SparderQueryPlanExec implements QueryPlanExec {
}
}
}
+
+ private void checkOnlyTsAnswer() {
+ if (QueryContext.current().getForcedToTieredStorage() == ForceToTieredStorage.CH_FAIL_TO_RETURN) {
+ throw new KylinException(QueryErrorCode.FORCED_TO_TIEREDSTORAGE_RETURN_ERROR,
+ MsgPicker.getMsg().getForcedToTieredstorageReturnError());
+ }
+ }
}
diff --git a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
index ffb4340233..006c03a3ee 100644
--- a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
+++ b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/SecondStorageLockTest.java
@@ -2511,7 +2511,9 @@ public class SecondStorageLockTest implements JobWaiter {
final String catalog = "default";
Unsafe.setProperty(ClickHouseLoad.SOURCE_URL, getSourceUrl());
Unsafe.setProperty(ClickHouseLoad.ROOT_PATH, getLocalWorkingDirectory());
- configClickhouseWith(new JdbcDatabaseContainer[] { clickhouse1, clickhouse2 }, 1, catalog, () -> {
+ val container = new JdbcDatabaseContainer[] { clickhouse1, clickhouse2 };
+ int replica = 1;
+ configClickhouseWith(container, 1, catalog, () -> {
secondStorageService.changeProjectSecondStorageState(getProject(),
SecondStorageNodeHelper.getAllPairs(), true);
Assert.assertEquals(2, SecondStorageUtil.listProjectNodes(getProject()).size());
@@ -2525,6 +2527,8 @@ public class SecondStorageLockTest implements JobWaiter {
String sql = "select CAL_DT from TEST_KYLIN_FACT where CAL_DT between '2012-01-01' and '2012-01-02'";
Map<String, Map<String, Boolean>> nodeStatusMap;
+ testForceToTSAndChDown(sql, container, replica);
+
{
// testGroupNodeDownForceToTierStorageOK
clearQueryContext();
@@ -2606,28 +2610,46 @@ public class SecondStorageLockTest implements JobWaiter {
}
triggerClickHouseJob(getDataFlow());
}
-
- {
- testReverseForceToTierStorageWhenCHUnavailable(sql);
- }
-
- {
- testReverseForceToTierStorageWhenCHOK(sql);
- }
-
+
+ testReverseForceToTierStorageWhenCHUnavailable(sql);
+ testReverseForceToTierStorageWhenCHOK(sql);
+
//reset status
nodeStatusMap = ImmutableMap.of("pair0", ImmutableMap.of("node00", true), "pair1",
ImmutableMap.of("node01", true));
secondStorageEndpoint.updateNodeStatus(nodeStatusMap);
- {
- testForceToTierStorageShutTierStorage(sql);
- }
-
+ testForceToTierStorageShutTierStorage(sql);
return true;
});
}
}
+
+ @SneakyThrows
+ private void testForceToTSAndChDown(String sql, JdbcDatabaseContainer<?>[] container, int replica) {
+ ExecAndComp.queryModel(getProject(), sql);
+ OLAPContext.getNativeRealizations().stream().findFirst().ifPresent(r -> assertTrue(r.isSecondStorage()));
+
+ for (JdbcDatabaseContainer<?> clickhouse : container) {
+ clickhouse.stop();
+ }
+ clearQueryContext();
+ QueryContext queryContext = QueryContext.current();
+ queryContext.setForcedToTieredStorage(ForceToTieredStorage.CH_FAIL_TO_RETURN);
+ queryContext.setForceTableIndex(true);
+ assertThrows(SQLException.class, () -> ExecAndComp.queryModel(getProject(), sql));
+
+ clearQueryContext();
+ queryContext = QueryContext.current();
+ queryContext.setForcedToTieredStorage(ForceToTieredStorage.CH_FAIL_TO_DFS);
+ queryContext.setForceTableIndex(false);
+ ExecAndComp.queryModel(getProject(), sql);
+
+ for (JdbcDatabaseContainer<?> clickhouse : container) {
+ clickhouse.start();
+ }
+ ClickHouseUtils.internalConfigClickHouse(container, replica);
+ }
private void testReverseForceToTierStorageWhenCHUnavailable(String sql) {
// testReverseForceToTierStorageWhenCHUnavailable