You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/05/06 06:59:09 UTC
[kylin] 08/38: KYLIN-5525 fix error when model init error in multi threads
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 09ea4572781fcc0ea9da76204e3fdce52d8f85c6
Author: ChenLiang.Lu <31...@users.noreply.github.com>
AuthorDate: Thu Feb 16 10:09:42 2023 +0800
KYLIN-5525 fix error when model init error in multi threads
---
.../kylin/metadata/model/ColExcludedChecker.java | 4 +++-
.../apache/kylin/metadata/model/NDataModel.java | 4 ++++
.../realization/RealizationRuntimeException.java | 28 ++++++++++++++++++++++
.../org/apache/kylin/newten/TableIndexTest.java | 19 +++++++++++++++
.../kylin/query/routing/RealizationChooser.java | 3 +++
5 files changed, 57 insertions(+), 1 deletion(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
index 8fd68f26af..97b3c3e8c6 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColExcludedChecker.java
@@ -88,7 +88,9 @@ public class ColExcludedChecker {
if (model == null || model.isBroken()) {
return;
}
- model.init(config, project, Lists.newArrayList());
+ if (!model.isInitAlready()) {
+ model.init(config, project, Lists.newArrayList());
+ }
model.getAllTables().stream().filter(Objects::nonNull) //
.flatMap(tableRef -> tableRef.getColumns().stream())
.filter(tblColRef -> excludedCols.contains(tblColRef.getColumnDesc()))
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
index c51b0cd5ff..fbb704a871 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/NDataModel.java
@@ -280,6 +280,9 @@ public class NDataModel extends RootPersistentEntity {
// mark this model as used for model save checking
private boolean saveCheck = false;
+ // mark this model has invoked init() function
+ private boolean initAlready = false;
+
public enum ColumnStatus {
TOMB, EXIST, DIMENSION
}
@@ -1026,6 +1029,7 @@ public class NDataModel extends RootPersistentEntity {
throw new KylinException(TABLE_JOIN_RELATIONSHIP_ERROR,
MsgPicker.getMsg().getDimensionTableUsedInThisModel());
}
+ this.setInitAlready(true);
}
public ComputedColumnUtil.CCConflictInfo checkCCFailAtEnd(KylinConfig config, String project,
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRuntimeException.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRuntimeException.java
new file mode 100644
index 0000000000..aff464f813
--- /dev/null
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRuntimeException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.realization;
+
+public class RealizationRuntimeException extends RuntimeException {
+
+ private static final long serialVersionUID = 7631508437415520091L;
+
+ public RealizationRuntimeException(String message, Throwable t) {
+ super(message, t);
+ }
+}
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/TableIndexTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/TableIndexTest.java
index 3c94b1f00c..1d9200f8d9 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/TableIndexTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/TableIndexTest.java
@@ -29,9 +29,12 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
+import org.apache.kylin.metadata.realization.RealizationRuntimeException;
import org.apache.kylin.util.ExecAndComp;
import org.apache.kylin.util.ExecAndComp.CompareLevel;
+
import org.apache.spark.sql.SparderEnv;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -172,4 +175,20 @@ public class TableIndexTest extends NLocalWithSparkSessionTest {
+ "WHERE KYLIN_FACT.cal_dt = DATE '2012-01-01' "));
ExecAndComp.execAndCompare(query, getProject(), CompareLevel.SAME, "left");
}
+
+ @Test
+ public void testUseTableIndexUnionQuery() throws Exception {
+ overwriteSystemProp("kylin.query.use-tableindex-answer-non-raw-query", "true");
+ overwriteSystemProp("kylin.metadata.table-exclusion-enabled", "true");
+ fullBuild("acfde546-2cc9-4eec-bc92-e3bd46d4e2ee");
+ populateSSWithCSVData(getTestConfig(), getProject(), SparderEnv.getSparkSession());
+ List<Pair<String, String>> query = new ArrayList<>();
+
+ query.add(Pair.newPair("query_table_index2", "select sum(PRICE) from TEST_KYLIN_FACT group by PRICE"
+ + " union all select sum(PRICE) from TEST_KYLIN_FACT group by PRICE"));
+ ExecAndComp.execAndCompare(query, getProject(), CompareLevel.SAME, "left");
+ RealizationRuntimeException error = new RealizationRuntimeException("unexpected error", new RuntimeException(
+ "error"));
+ assert error.getMessage().contains("unexpected error");
+ }
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
index d4f70edd52..389eb8b074 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/routing/RealizationChooser.java
@@ -95,6 +95,7 @@ import org.apache.kylin.metadata.realization.CapabilityResult;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.NoRealizationFoundException;
import org.apache.kylin.metadata.realization.NoStreamingRealizationFoundException;
+import org.apache.kylin.metadata.realization.RealizationRuntimeException;
import org.apache.kylin.metadata.realization.SQLDigest;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.relnode.OLAPContextProp;
@@ -185,6 +186,8 @@ public class RealizationChooser {
throw (NoRealizationFoundException) e.getCause();
} else if (e.getCause() instanceof NoStreamingRealizationFoundException) {
throw (NoStreamingRealizationFoundException) e.getCause();
+ } else {
+ throw new RealizationRuntimeException("unexpected error when choose layout", e);
}
} catch (InterruptedException e) {
for (Future<?> future : futureList) {