[kylin] 03/12: KYLIN-2894 add a new signature calculator FactTableRealizationSetCalculator

[sh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit 7e1cdeda85dc293efe8bd408444d7bbef8401be6
Author: Ma,Gang <ga...@ebay.com>
AuthorDate: Thu Oct 18 17:41:37 2018 +0800

    KYLIN-2894 add a new signature calculator FactTableRealizationSetCalculator
---
 .../org/apache/kylin/common/KylinConfigBase.java   |   2 +-
 .../FactTableRealizationSetCalculator.java         | 112 +++++++++++++++++++++
 .../kylin/rest/util/SQLResponseSignatureUtil.java  |   6 +-
 3 files changed, 116 insertions(+), 4 deletions(-)

diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 43c0831..b19f2e9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1555,7 +1555,7 @@ abstract public class KylinConfigBase implements Serializable {
 
     public String getSQLResponseSignatureClass() {
         return this.getOptional("kylin.query.signature-class",
-                "org.apache.kylin.rest.signature.RealizationSetCalculator");
+                "org.apache.kylin.rest.signature.FactTableRealizationSetCalculator");
     }
 
     public boolean isQueryCacheSignatureEnabled() {
diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/FactTableRealizationSetCalculator.java b/server-base/src/main/java/org/apache/kylin/rest/signature/FactTableRealizationSetCalculator.java
new file mode 100644
index 0000000..944cb1b
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/signature/FactTableRealizationSetCalculator.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.signature;
+
+import java.util.Set;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+import org.apache.kylin.storage.hybrid.HybridManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Sets;
+
+public class FactTableRealizationSetCalculator extends RealizationSetCalculator {
+
+    public static final Logger logger = LoggerFactory.getLogger(FactTableRealizationSetCalculator.class);
+
+    /**
+     * In case that cube selection result changes after a new cube's data is ready,
+     * the cache result should be invalidated, which requires the related signature should be changed.
+     * To achieve this, we need to consider all of those cubes who shares the same fact table
+     */
+    @Override
+    protected Set<String> getRealizations(KylinConfig config, String cubes, ProjectInstance project) {
+        Set<String> realizations = super.getRealizations(config, cubes, project);
+        if (realizations == null) {
+            return null;
+        }
+        Set<String> factTables = Sets.newHashSet();
+        for (String realName : realizations) {
+            IRealization realInstance = getRealization(config, realName);
+            String factTable = getRootFactTableForRealization(realInstance);
+            if (factTable != null) {
+                factTables.add(factTable);
+            }
+        }
+
+        Set<String> ret = Sets.newHashSet(realizations);
+        for (RealizationEntry entry : project.getRealizationEntries()) {
+            String realName = entry.getRealization();
+            IRealization realInstance = getRealization(config, realName, entry.getType());
+            String factTableForEntry = getRootFactTableForRealization(realInstance);
+            if (factTableForEntry != null) {
+                if (factTables.contains(factTableForEntry)) {
+                    ret.add(realName);
+                }
+            }
+        }
+        return ret;
+    }
+
+    private String getRootFactTableForRealization(IRealization realization) {
+        if (realization == null) {
+            logger.warn("Cannot find realization %s", realization);
+            return null;
+        }
+        DataModelDesc model = realization.getModel();
+        if (model == null) {
+            logger.warn("The model for realization %s is null", realization.getName());
+            return null;
+        }
+        TableRef rootFactTable = model.getRootFactTable();
+        if (rootFactTable == null) {
+            logger.warn("The root table for model %s is null", model.getName());
+            return null;
+        }
+        return rootFactTable.getTableIdentity();
+    }
+
+    private IRealization getRealization(KylinConfig config, String name, RealizationType type) {
+        switch (type) {
+        case CUBE:
+            return CubeManager.getInstance(config).getCube(name);
+        case HYBRID:
+            return HybridManager.getInstance(config).getHybridInstance(name);
+        default:
+            return getRealization(config, name);
+        }
+    }
+
+    private IRealization getRealization(KylinConfig config, String name) {
+        HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(name);
+        if (hybridInstance != null) {
+            return hybridInstance;
+        }
+        return CubeManager.getInstance(config).getCube(name);
+    }
+}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java
index c6d3507..2a57554 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java
@@ -22,7 +22,7 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.rest.response.SQLResponse;
-import org.apache.kylin.rest.signature.RealizationSetCalculator;
+import org.apache.kylin.rest.signature.FactTableRealizationSetCalculator;
 import org.apache.kylin.rest.signature.SignatureCalculator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -52,7 +52,7 @@ public class SQLResponseSignatureUtil {
             signatureCalculator = (SignatureCalculator) signatureClass.getConstructor().newInstance();
         } catch (Exception e) {
             logger.warn("Will use default signature since fail to construct signature due to " + e);
-            signatureCalculator = new RealizationSetCalculator();
+            signatureCalculator = new FactTableRealizationSetCalculator();
         }
         return signatureCalculator.calculateSignature(config, sqlResponse, project);
     }
@@ -62,7 +62,7 @@ public class SQLResponseSignatureUtil {
             return Class.forName(config.getSQLResponseSignatureClass());
         } catch (ClassNotFoundException e) {
             logger.warn("Will use default signature since cannot find class " + config.getSQLResponseSignatureClass());
-            return RealizationSetCalculator.class;
+            return FactTableRealizationSetCalculator.class;
         }
     }
 }