You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by "zhouyuan (via GitHub)" <gi...@apache.org> on 2026/10/19 16:35:30 UTC

Re: [PR] [GLUTEN-10134][Vl] elt ansi function overlay [gluten]

zhouyuan commented on code in PR #12873:
URL: https://github.com/apache/gluten/pull/12873#discussion_r4316316490


##########
cpp/velox/tests/SparkFunctionTest.cc:
##########
@@ -144,3 +151,189 @@ TEST_F(SparkFunctionTest, expressionLevelLegacyCastIgnoresSessionAnsiOn) {
 
   facebook::velox::test::assertEqualVectors(makeFlatVector<int8_t>({-121}), evaluate(legacyCast, input));
 }
+
+TEST_F(SparkFunctionTest, elt) {
+  // The index picks a different input per row. A NULL in an input that is not
+  // picked does not affect the result, while a NULL in the picked one does.
+  auto index = makeFlatVector<int32_t>({1, 2, 3, 3});
+  auto first = makeNullableFlatVector<StringView>({"a0", std::nullopt, "a2", "a3"});
+  auto second = makeNullableFlatVector<StringView>({std::nullopt, "b1", "b2", "b3"});
+  auto third = makeNullableFlatVector<StringView>({"c0", "c1", "c2", std::nullopt});
+
+  facebook::velox::test::assertEqualVectors(
+      makeNullableFlatVector<StringView>({"a0", "b1", "c2", std::nullopt}),
+      evaluate("elt(c0, c1, c2, c3)", makeRowVector({index, first, second, third})));
+}
+
+TEST_F(SparkFunctionTest, eltNullIndex) {
+  // A NULL index returns NULL, with ANSI mode on as well.
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "true"}});
+  auto index = makeNullableFlatVector<int32_t>({std::nullopt, 2});
+  auto first = makeFlatVector<StringView>({"a0", "a1"});
+  auto second = makeFlatVector<StringView>({"b0", "b1"});
+
+  facebook::velox::test::assertEqualVectors(
+      makeNullableFlatVector<StringView>({std::nullopt, "b1"}),
+      evaluate("elt(c0, c1, c2)", makeRowVector({index, first, second})));
+}
+
+TEST_F(SparkFunctionTest, eltIndexOutOfRangeAnsiOff) {
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "false"}});
+  auto index = makeFlatVector<int32_t>({0, -1, 3, 2});
+  auto first = makeFlatVector<StringView>({"a0", "a1", "a2", "a3"});
+  auto second = makeFlatVector<StringView>({"b0", "b1", "b2", "b3"});
+
+  facebook::velox::test::assertEqualVectors(
+      makeNullableFlatVector<StringView>({std::nullopt, std::nullopt, std::nullopt, "b3"}),
+      evaluate("elt(c0, c1, c2)", makeRowVector({index, first, second})));
+}
+
+TEST_F(SparkFunctionTest, eltIndexOutOfRangeAnsiOn) {
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "true"}});
+  auto first = makeFlatVector<StringView>({"a0", "a1"});
+  auto second = makeFlatVector<StringView>({"b0", "b1"});
+
+  auto evaluateWithIndex = [&](const std::vector<int32_t>& indexes) {
+    return evaluate("elt(c0, c1, c2)", makeRowVector({makeFlatVector<int32_t>(indexes), first, second}));
+  };
+
+  VELOX_ASSERT_THROW(evaluateWithIndex({1, 0}), "The index is out of bounds for elt. index: 0, number of inputs: 2");
+  VELOX_ASSERT_THROW(evaluateWithIndex({-1, 1}), "The index is out of bounds for elt. index: -1, number of inputs: 2");
+  VELOX_ASSERT_THROW(evaluateWithIndex({1, 3}), "The index is out of bounds for elt. index: 3, number of inputs: 2");
+
+  // In-range indexes are unaffected by ANSI mode.
+  facebook::velox::test::assertEqualVectors(makeFlatVector<StringView>({"a0", "b1"}), evaluateWithIndex({1, 2}));
+}
+
+TEST_F(SparkFunctionTest, eltVarbinary) {
+  auto index = makeFlatVector<int32_t>({2, 1});
+  auto first = makeNullableFlatVector<StringView>({"a0", "a1"}, VARBINARY());
+  auto second = makeNullableFlatVector<StringView>({"b0", std::nullopt}, VARBINARY());
+
+  auto result = evaluate("elt(c0, c1, c2)", makeRowVector({index, first, second}));
+  ASSERT_EQ(result->type()->kind(), TypeKind::VARBINARY);
+  facebook::velox::test::assertEqualVectors(makeNullableFlatVector<StringView>({"b0", "a1"}, VARBINARY()), result);
+}
+
+TEST_F(SparkFunctionTest, eltConstantIndexOverDictionaryInput) {
+  // A constant index selects the same input for all rows, and the selected
+  // input may carry an encoding.
+  auto index = makeConstant<int32_t>(2, 3);
+  auto first = makeFlatVector<StringView>({"a0", "a1", "a2"});
+  auto second = makeFlatVector<StringView>({"b0", "b1", "b2"});
+  auto dictionary = BaseVector::wrapInDictionary(nullptr, makeIndices({2, 0, 1}), 3, second);
+
+  facebook::velox::test::assertEqualVectors(
+      makeFlatVector<StringView>({"b2", "b0", "b1"}),
+      evaluate("elt(c0, c1, c2)", makeRowVector({index, first, dictionary})));
+}
+
+TEST_F(SparkFunctionTest, convOverflowAnsiOff) {
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "false"}});
+
+  // Digits that do not fit in an unsigned 64-bit integer saturate, which is
+  // what Spark does with ANSI mode off.
+  EXPECT_EQ(conv("9223372036854775807", 36, 16), "FFFFFFFFFFFFFFFF");
+  EXPECT_EQ(conv("10000000000000000", 16, 10), "18446744073709551615");
+  EXPECT_EQ(conv("-10000000000000000", 16, -10), "-1");
+}
+
+TEST_F(SparkFunctionTest, convOverflowAnsiOn) {
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "true"}});
+
+  VELOX_ASSERT_THROW(conv("9223372036854775807", 36, 16), "Overflow in function conv()");
+  VELOX_ASSERT_THROW(conv("10000000000000000", 16, 10), "Overflow in function conv()");
+  VELOX_ASSERT_THROW(conv("-10000000000000000", 16, -10), "Overflow in function conv()");
+}
+
+TEST_F(SparkFunctionTest, convNoOverflowAnsiOn) {
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "true"}});
+
+  EXPECT_EQ(conv("4", 10, 2), "100");
+  EXPECT_EQ(conv("big", 36, 16), "3A48");
+  // 2^64 - 1 is the largest input that does not overflow.
+  EXPECT_EQ(conv("18446744073709551615", 10, 10), "18446744073709551615");
+  EXPECT_EQ(conv("FFFFFFFFFFFFFFFF", 16, -10), "-1");
+  // The sign is applied after the digits are accumulated, so wrapping around
+  // through a negative input is not an overflow.
+  EXPECT_EQ(conv("-1", 10, 16), "FFFFFFFFFFFFFFFF");
+  EXPECT_EQ(conv("-15", 10, 16), "FFFFFFFFFFFFFFF1");
+  // The digits stop at the first character that is invalid for the base, so a
+  // long tail of invalid characters is not an overflow either.
+  EXPECT_EQ(conv("11abcabcabcabcabcabcabcabc", 10, 16), "B");
+}
+
+TEST_F(SparkFunctionTest, convInvalidInputAnsiOn) {
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "true"}});
+
+  // Invalid input gives NULL, with ANSI mode on as well.
+  EXPECT_EQ(conv("15", 1, 10), std::nullopt);
+  EXPECT_EQ(conv("15", 37, 10), std::nullopt);
+  EXPECT_EQ(conv("15", 10, 1), std::nullopt);
+  EXPECT_EQ(conv("15", 10, -37), std::nullopt);
+  EXPECT_EQ(conv("", 10, 16), std::nullopt);
+  EXPECT_EQ(conv("   ", 10, 16), std::nullopt);
+  EXPECT_EQ(conv(std::nullopt, 10, 16), std::nullopt);
+}
+
+TEST_F(SparkFunctionTest, elementAtArrayOutOfBoundAnsiOff) {
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "false"}});
+  auto array = makeArrayVector<int32_t>({{1, 2, 3}, {1, 2, 3}, {1, 2, 3}, {1, 2, 3}, {1, 2, 3}});
+  auto index = makeFlatVector<int32_t>({1, 3, -1, 4, -4});
+
+  // An index past either end of the array gives NULL with ANSI mode off.
+  facebook::velox::test::assertEqualVectors(
+      makeNullableFlatVector<int32_t>({1, 3, 3, std::nullopt, std::nullopt}),
+      evaluate("element_at(c0, c1)", makeRowVector({array, index})));
+}
+
+TEST_F(SparkFunctionTest, elementAtArrayOutOfBoundAnsiOn) {
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "true"}});
+  auto array = makeArrayVector<int32_t>({{1, 2, 3}, {1, 2, 3}});
+
+  // In-bound indices, including the negative ones counting from the end, are
+  // unaffected by ANSI mode.
+  facebook::velox::test::assertEqualVectors(
+      makeFlatVector<int32_t>({1, 3}),
+      evaluate("element_at(c0, c1)", makeRowVector({array, makeFlatVector<int32_t>({1, -1})})));
+
+  VELOX_ASSERT_THROW(
+      evaluate("element_at(c0, c1)", makeRowVector({array, makeFlatVector<int32_t>({1, 4})})),
+      "Array subscript index out of bounds");
+  VELOX_ASSERT_THROW(
+      evaluate("element_at(c0, c1)", makeRowVector({array, makeFlatVector<int32_t>({-4, 1})})),
+      "Array subscript index out of bounds");
+}
+
+TEST_F(SparkFunctionTest, elementAtArrayZeroIndex) {
+  auto data = makeRowVector({makeArrayVector<int32_t>({{1, 2, 3}}), makeFlatVector<int32_t>({0})});
+
+  // An index of 0 is an error whatever the ANSI mode is.
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "false"}});
+  VELOX_ASSERT_THROW(evaluate("element_at(c0, c1)", data), "SQL array indices start at 1");
+
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "true"}});
+  VELOX_ASSERT_THROW(evaluate("element_at(c0, c1)", data), "SQL array indices start at 1");
+}
+
+TEST_F(SparkFunctionTest, elementAtMapMissingKeyAnsiOn) {
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "true"}});
+  auto map = makeMapVector<int32_t, int32_t>({{{1, 10}, {2, 20}}, {{1, 10}, {2, 20}}});
+  auto key = makeFlatVector<int32_t>({2, 3});
+
+  // A key the map does not contain gives NULL, with ANSI mode on as well.
+  facebook::velox::test::assertEqualVectors(
+      makeNullableFlatVector<int32_t>({20, std::nullopt}), evaluate("element_at(c0, c1)", makeRowVector({map, key})));
+}
+
+TEST_F(SparkFunctionTest, sizeOfNull) {
+  // Gluten passes Spark's Size.legacySizeOfNull as the second argument, and
+  // Spark derives it from 'spark.sql.legacy.sizeOfNull' AND NOT ANSI mode, so
+  // both behaviors have to work regardless of the session's ANSI mode.
+  queryCtx_->testingOverrideConfigUnsafe({{sparkAnsiEnabledConfigKey(), "true"}});

Review Comment:
   removed



##########
backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala:
##########
@@ -1251,6 +1251,68 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with Logging {
     GenericExpressionTransformer(substraitExprName, child, expr)
   }
 
+  override def genEltTransformer(
+      substraitExprName: String,
+      children: Seq[ExpressionTransformer],
+      expr: Elt): ExpressionTransformer = {
+    // Velox's elt derives whether an out-of-range index raises an error from the session's
+    // 'spark.sql.ansi.enabled', while Spark captures it in Elt.failOnError at analysis time.
+    // The two normally agree; fall back when they don't, so the ANSI behavior never diverges.
+    if (expr.failOnError != SQLConf.get.ansiEnabled) {
+      GlutenExceptionUtil
+        .throwsNotFullySupported(
+          ExpressionNames.ELT,
+          EltRestrictions.NOT_SUPPORT_FAIL_ON_ERROR_MISMATCH
+        )
+    }
+    GenericExpressionTransformer(substraitExprName, children, expr)
+  }
+
+  override def genConvTransformer(
+      substraitExprName: String,
+      children: Seq[ExpressionTransformer],
+      expr: Conv): ExpressionTransformer = {
+    // Velox derives whether an overflow raises an error from the session's
+    // 'spark.sql.ansi.enabled', while Spark captures it in Conv.ansiEnabled at analysis time.
+    // The two normally agree; fall back when they don't, so the ANSI behavior never diverges.
+    if (expr.ansiEnabled != SQLConf.get.ansiEnabled) {
+      GlutenExceptionUtil
+        .throwsNotFullySupported(
+          ExpressionNames.CONV,
+          ConvRestrictions.NOT_SUPPORT_ANSI_ENABLED_MISMATCH
+        )
+    }
+    GenericExpressionTransformer(substraitExprName, children, expr)
+  }
+
+  override def genElementAtTransformer(
+      substraitExprName: String,
+      children: Seq[ExpressionTransformer],
+      expr: ElementAt): ExpressionTransformer = {
+    // Only the array input reads failOnError: Spark returns NULL for a key a map does not
+    // contain whatever the ANSI mode is, and so does Velox.
+    if (expr.left.dataType.isInstanceOf[ArrayType]) {
+      if (expr.defaultValueOutOfBound.isDefined) {
+        GlutenExceptionUtil
+          .throwsNotFullySupported(
+            ExpressionNames.ELEMENT_AT,
+            ElementAtRestrictions.NOT_SUPPORT_DEFAULT_VALUE_OUT_OF_BOUND
+          )
+      }
+      // Velox derives whether an out-of-bound index raises an error from the session's
+      // 'spark.sql.ansi.enabled', while Spark captures it in ElementAt.failOnError at
+      // analysis time. The two normally agree; fall back when they don't.
+      if (expr.failOnError != SQLConf.get.ansiEnabled) {
+        GlutenExceptionUtil
+          .throwsNotFullySupported(
+            ExpressionNames.ELEMENT_AT,
+            ElementAtRestrictions.NOT_SUPPORT_FAIL_ON_ERROR_MISMATCH

Review Comment:
   fixed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org