You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by "rui-mo (via GitHub)" <gi...@apache.org> on 2024/06/04 12:28:56 UTC

Re: [PR] [GLUTEN-5965][VL] Support pushdown "not in" to scan node [incubator-gluten]

rui-mo commented on code in PR #5966:
URL: https://github.com/apache/incubator-gluten/pull/5966#discussion_r1625881664


##########
cpp/velox/substrait/SubstraitToVeloxPlan.cc:
##########
@@ -1449,10 +1450,14 @@ connector::hive::SubfieldFilters SubstraitToVeloxPlanConverter::createSubfieldFi
       auto expr = scalarFunction.arguments()[0].value();
       if (expr.has_scalar_function()) {
         // Set its child to filter info with reverse enabled.
-        setFilterInfo(scalarFunction.arguments()[0].value().scalar_function(), inputTypeList, columnToFilterInfo, true);
+        setFilterInfo(expr.scalar_function(), inputTypeList, columnToFilterInfo, true);
+      } else if (expr.has_singular_or_list()) {
+        // Since currently only integer and string types in "IN" expression
+        // can be pushed down, Options types should be checked

Review Comment:
   > Options types should be checked
   
   Could you remind me where the check for types is added?



##########
cpp/velox/substrait/SubstraitToVeloxPlan.cc:
##########
@@ -1577,26 +1582,28 @@ bool SubstraitToVeloxPlanConverter::canPushdownNot(
     std::vector<RangeRecorder>& rangeRecorders) {
   VELOX_CHECK(scalarFunction.arguments().size() == 1, "Only one arg is expected for Not.");
   const auto& notArg = scalarFunction.arguments()[0];
-  if (!notArg.value().has_scalar_function()) {
-    // Not for a Boolean Literal or Or List is not supported curretly.
-    // It can be pushed down with an AlwaysTrue or AlwaysFalse Range.
-    return false;
-  }
-
-  auto argFunction =
-      SubstraitParser::findFunctionSpec(functionMap_, notArg.value().scalar_function().function_reference());
-  auto functionName = SubstraitParser::getNameBeforeDelimiter(argFunction);
+  if (notArg.value().has_singular_or_list()) {
+    auto singularOrList = notArg.value().singular_or_list();
+    uint32_t colIdx = getColumnIndexFromSingularOrList(singularOrList);
+    return rangeRecorders.at(colIdx).setInRange();
+  } else if (notArg.value().has_scalar_function()) {
+    auto argFunction =
+        SubstraitParser::findFunctionSpec(functionMap_, notArg.value().scalar_function().function_reference());
+    auto functionName = SubstraitParser::getNameBeforeDelimiter(argFunction);
 
-  static const std::unordered_set<std::string> supportedNotFunctions = {sGte, sGt, sLte, sLt, sEqual};
+    static const std::unordered_set<std::string> supportedNotFunctions = {sGte, sGt, sLte, sLt, sEqual};
 
-  uint32_t fieldIdx;
-  bool isFieldOrWithLiteral = fieldOrWithLiteral(notArg.value().scalar_function().arguments(), fieldIdx);
+    uint32_t fieldIdx;
+    bool isFieldOrWithLiteral = fieldOrWithLiteral(notArg.value().scalar_function().arguments(), fieldIdx);
 
-  if (supportedNotFunctions.find(functionName) != supportedNotFunctions.end() && isFieldOrWithLiteral &&
-      rangeRecorders.at(fieldIdx).setCertainRangeForFunction(functionName, true /*reverse*/)) {
-    return true;
+    if (supportedNotFunctions.find(functionName) != supportedNotFunctions.end() && isFieldOrWithLiteral &&
+        rangeRecorders.at(fieldIdx).setCertainRangeForFunction(functionName, true /*reverse*/)) {
+      return true;
+    }
+    return false;

Review Comment:
   This return seems to be not needed as the else branch will return false anyway.



##########
cpp/velox/substrait/SubstraitToVeloxPlan.h:
##########
@@ -364,9 +364,11 @@ class SubstraitToVeloxPlanConverter {
       }
     }
 
-    // Set a value for the not(equal) condition.
-    void setNotValue(const std::optional<variant>& notValue) {
-      notValue_ = notValue;
+    // Set a list of values to be used in the push down of 'not in' expression.
+    void setNotValue(const std::vector<variant>& notValue) {
+      for (const auto& value : notValue) {
+        notValues_.emplace_back(value);
+      }

Review Comment:
   nit: suggestion on the namings: notValue -> notValues, setNotValue -> setNotValues



##########
cpp/velox/substrait/SubstraitToVeloxPlan.cc:
##########
@@ -2118,44 +2107,25 @@ void SubstraitToVeloxPlanConverter::constructSubfieldFilters(
   } else {
     using NativeType = typename RangeTraits<KIND>::NativeType;
     using RangeType = typename RangeTraits<KIND>::RangeType;
-    using MultiRangeType = typename RangeTraits<KIND>::MultiRangeType;
 
     // Handle 'in' filter.
     if (filterInfo.values_.size() > 0) {
       // To filter out null is a default behaviour of Spark IN expression.
       nullAllowed = false;
-      setInFilter<KIND>(filterInfo.values_, nullAllowed, inputName, filters);
+      setInFilter<KIND>(filterInfo.values_, nullAllowed, false, inputName, filters);
       // Currently, In cannot coexist with other filter conditions
       // due to multirange is in 'OR' relation but 'AND' is needed.
       VELOX_CHECK(rangeSize == 0, "LowerBounds or upperBounds conditons cannot be supported after IN filter.");
-      VELOX_CHECK(!filterInfo.notValue_.has_value(), "Not equal cannot be supported after IN filter.");
+      VELOX_CHECK(filterInfo.notValues_.size() == 0, "Not equal cannot be supported after IN filter.");
       return;
     }
 
-    // Construct the Filters.
-    std::vector<std::unique_ptr<FilterType>> colFilters;
-
-    // Handle not(equal) filter.
-    if (filterInfo.notValue_) {
-      variant notVariant = filterInfo.notValue_.value();
-      createNotEqualFilter<KIND, FilterType>(notVariant, filterInfo.nullAllowed_, colFilters);
+    // Handle not in filter.
+    if (filterInfo.notValues_.size() > 0) {
+      setInFilter<KIND>(filterInfo.notValues_, filterInfo.nullAllowed_, true, inputName, filters);
       // Currently, Not-equal cannot coexist with other filter conditions
       // due to multirange is in 'OR' relation but 'AND' is needed.
       VELOX_CHECK(rangeSize == 0, "LowerBounds or upperBounds conditons cannot be supported after not-equal filter.");

Review Comment:
   Could you also update the comment and error message as this branch handles more cases than not-equal?



##########
backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala:
##########
@@ -156,6 +156,64 @@ class TestOperator extends VeloxWholeStageTransformerSuite {
     checkLengthAndPlan(df, 60141)
   }
 
+  test("not in") {
+    // integral type
+    val df = runQueryAndCompare(
+      "select l_orderkey from lineitem " +
+        "where l_partkey not in (1552, 674, 1062)") {
+      checkGlutenOperatorMatch[FileSourceScanExecTransformer]
+    }
+    checkLengthAndPlan(df, 60053)
+
+    val df2 = runQueryAndCompare(
+      "select l_orderkey from lineitem " +
+        "where l_partkey not in (1552, 674) and l_partkey not in (1062)") {
+      checkGlutenOperatorMatch[FileSourceScanExecTransformer]
+    }
+    checkLengthAndPlan(df2, 60053)
+
+    val df3 = runQueryAndCompare(
+      "select l_orderkey from lineitem " +
+        "where l_partkey not in (1552, 674) and l_partkey != 1062") {
+      checkGlutenOperatorMatch[FileSourceScanExecTransformer]
+    }
+    checkLengthAndPlan(df3, 60053)
+
+    // string type
+    val df4 =
+      runQueryAndCompare("select o_orderstatus from orders where o_orderstatus not in ('O', 'F')") {
+        checkGlutenOperatorMatch[FileSourceScanExecTransformer]
+      }
+    checkLengthAndPlan(df4, 363)
+
+    // bool type
+    withTable("t") {
+      sql("create table t (id int, b boolean) using parquet")
+      sql("insert into t values (1, true), (2, false), (3, null)")
+      runQueryAndCompare("select * from t where b not in (true)") {
+        checkGlutenOperatorMatch[FileSourceScanExecTransformer]
+      }
+
+      runQueryAndCompare("select * from t where b not in (true, false)") {
+        checkGlutenOperatorMatch[FileSourceScanExecTransformer]
+      }
+    }
+
+    // mix not-in with range
+    runQueryAndCompare(
+      "select l_orderkey from lineitem " +
+        "where l_partkey not in (1552, 674) and l_partkey >= 1552") {
+      checkGlutenOperatorMatch[FileSourceScanExecTransformer]
+    }
+
+    // mix not-in with in
+    runQueryAndCompare(
+      "select l_orderkey from lineitem " +
+        "where l_partkey not in (1552, 674) and l_partkey in (1552)") {
+      checkGlutenOperatorMatch[FileSourceScanExecTransformer]

Review Comment:
   Could we add a test for or relation like `l_partkey not in (1552, 674) or l_partkey in (1552)`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org