You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by "zhztheplayer (via GitHub)" <gi...@apache.org> on 2024/04/17 04:07:09 UTC
[PR] [VL] Add a bad test case when bloom_filter_agg is fallen back while might_contain is not [incubator-gluten]
zhztheplayer opened a new pull request, #5433:
URL: https://github.com/apache/incubator-gluten/pull/5433
So far the case fails with error
```
org.apache.gluten.exception.GlutenException: java.lang.RuntimeException: Exception: VeloxUserError
Error Source: USER
Error Code: INVALID_ARGUMENT
Reason: (1 vs. 0)
Retriable: False
Expression: kBloomFilterV1 == version
Function: merge
File: ../.././velox/common/base/BloomFilter.h
Line: 67
Stack trace:
# 0 std::shared_ptr<facebook::velox::VeloxException::State const> facebook::velox::VeloxException::State::make<facebook::velox::VeloxException::make(char const*, unsigned long, char const*, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, bool, facebook::velox::VeloxException::Type, std::basic_string_view<char, std::char_traits<char> >)::{lambda(auto:1&)#1}>(facebook::velox::VeloxException::Type, facebook::velox::VeloxException::make(char const*, unsigned long, char const*, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, bool, facebook::velox::VeloxException::Type, std::basic_string_view<char, std::char_traits<char> >)::{lambda(auto:1&)#1})
# 1 facebook::velox::VeloxException::VeloxException(char const*, unsigned long, char const*, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, bool, facebook::velox::VeloxException::Type, std::basic_string_view<char, std::char_traits<char> >)
# 2 facebook::velox::VeloxUserError::VeloxUserError(char const*, unsigned long, char const*, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >, bool, std::basic_string_view<char, std::char_traits<char> >)
# 3 void facebook::velox::detail::veloxCheckFail<facebook::velox::VeloxUserError, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&>(facebook::velox::detail::VeloxCheckFailArgs const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
# 4 facebook::velox::BloomFilter<std::allocator<unsigned long> >::merge(char const*)
# 5 facebook::velox::functions::sparksql::BloomFilterMightContainFunction<facebook::velox::exec::VectorExec>::initialize(std::vector<std::shared_ptr<facebook::velox::Type const>, std::allocator<std::shared_ptr<facebook::velox::Type const> > > const&, facebook::velox::core::QueryConfig const&, facebook::velox::StringView const*, long const*)
# 6 void facebook::velox::exec::SimpleFunctionAdapter<facebook::velox::core::UDFHolder<facebook::velox::functions::sparksql::BloomFilterMightContainFunction<facebook::velox::exec::VectorExec>, facebook::velox::exec::VectorExec, bool, facebook::velox::ConstantChecker<facebook::velox::Varbinary, long>, facebook::velox::Varbinary, long> >::unpackInitialize<2, facebook::velox::StringView, long>(std::vector<std::shared_ptr<facebook::velox::Type const>, std::allocator<std::shared_ptr<facebook::velox::Type const> > > const&, facebook::velox::core::QueryConfig const&, std::vector<std::shared_ptr<facebook::velox::BaseVector>, std::allocator<std::shared_ptr<facebook::velox::BaseVector> > > const&, facebook::velox::StringView const*, long const*) const
# 7 void facebook::velox::exec::SimpleFunctionAdapter<facebook::velox::core::UDFHolder<facebook::velox::functions::sparksql::BloomFilterMightContainFunction<facebook::velox::exec::VectorExec>, facebook::velox::exec::VectorExec, bool, facebook::velox::ConstantChecker<facebook::velox::Varbinary, long>, facebook::velox::Varbinary, long> >::unpackInitialize<1, facebook::velox::StringView>(std::vector<std::shared_ptr<facebook::velox::Type const>, std::allocator<std::shared_ptr<facebook::velox::Type const> > > const&, facebook::velox::core::QueryConfig const&, std::vector<std::shared_ptr<facebook::velox::BaseVector>, std::allocator<std::shared_ptr<facebook::velox::BaseVector> > > const&, facebook::velox::StringView const*) const
# 8 void facebook::velox::exec::SimpleFunctionAdapter<facebook::velox::core::UDFHolder<facebook::velox::functions::sparksql::BloomFilterMightContainFunction<facebook::velox::exec::VectorExec>, facebook::velox::exec::VectorExec, bool, facebook::velox::ConstantChecker<facebook::velox::Varbinary, long>, facebook::velox::Varbinary, long> >::unpackInitialize<0>(std::vector<std::shared_ptr<facebook::velox::Type const>, std::allocator<std::shared_ptr<facebook::velox::Type const> > > const&, facebook::velox::core::QueryConfig const&, std::vector<std::shared_ptr<facebook::velox::BaseVector>, std::allocator<std::shared_ptr<facebook::velox::BaseVector> > > const&) const
# 9 facebook::velox::exec::SimpleFunctionAdapter<facebook::velox::core::UDFHolder<facebook::velox::functions::sparksql::BloomFilterMightContainFunction<facebook::velox::exec::VectorExec>, facebook::velox::exec::VectorExec, bool, facebook::velox::ConstantChecker<facebook::velox::Varbinary, long>, facebook::velox::Varbinary, long> >::SimpleFunctionAdapter(std::vector<std::shared_ptr<facebook::velox::Type const>, std::allocator<std::shared_ptr<facebook::velox::Type const> > > const&, facebook::velox::core::QueryConfig const&, std::vector<std::shared_ptr<facebook::velox::BaseVector>, std::allocator<std::shared_ptr<facebook::velox::BaseVector> > > const&)
# 10 std::_MakeUniq<facebook::velox::exec::SimpleFunctionAdapter<facebook::velox::core::UDFHolder<facebook::velox::functions::sparksql::BloomFilterMightContainFunction<facebook::velox::exec::VectorExec>, facebook::velox::exec::VectorExec, bool, facebook::velox::ConstantChecker<facebook::velox::Varbinary, long>, facebook::velox::Varbinary, long> > >::__single_object std::make_unique<facebook::velox::exec::SimpleFunctionAdapter<facebook::velox::core::UDFHolder<facebook::velox::functions::sparksql::BloomFilterMightContainFunction<facebook::velox::exec::VectorExec>, facebook::velox::exec::VectorExec, bool, facebook::velox::ConstantChecker<facebook::velox::Varbinary, long>, facebook::velox::Varbinary, long> >, std::vector<std::shared_ptr<facebook::velox::Type const>, std::allocator<std::shared_ptr<facebook::velox::Type const> > > const&, facebook::velox::core::QueryConfig const&, std::vector<std::shared_ptr<facebook::velox::BaseVector>, std::allocator<std::shared_ptr<facebook::velox::B
aseVector> > > const&>(std::vector<std::shared_ptr<facebook::velox::Type const>, std::allocator<std::shared_ptr<facebook::velox::Type const> > > const&, facebook::velox::core::QueryConfig const&, std::vector<std::shared_ptr<facebook::velox::BaseVector>, std::allocator<std::shared_ptr<facebook::velox::BaseVector> > > const&)
# 11 facebook::velox::exec::SimpleFunctionAdapterFactoryImpl<facebook::velox::core::UDFHolder<facebook::velox::functions::sparksql::BloomFilterMightContainFunction<facebook::velox::exec::VectorExec>, facebook::velox::exec::VectorExec, bool, facebook::velox::ConstantChecker<facebook::velox::Varbinary, long>, facebook::velox::Varbinary, long> >::createVectorFunction(std::vector<std::shared_ptr<facebook::velox::Type const>, std::allocator<std::shared_ptr<facebook::velox::Type const> > > const&, std::vector<std::shared_ptr<facebook::velox::BaseVector>, std::allocator<std::shared_ptr<facebook::velox::BaseVector> > > const&, facebook::velox::core::QueryConfig const&) const
# 12 facebook::velox::exec::(anonymous namespace)::compileRewrittenExpression(std::shared_ptr<facebook::velox::core::ITypedExpr const> const&, facebook::velox::exec::(anonymous namespace)::Scope*, facebook::velox::core::QueryConfig const&, facebook::velox::memory::MemoryPool*, std::unordered_set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, bool)
# 13 facebook::velox::exec::(anonymous namespace)::compileExpression(std::shared_ptr<facebook::velox::core::ITypedExpr const> const&, facebook::velox::exec::(anonymous namespace)::Scope*, facebook::velox::core::QueryConfig const&, facebook::velox::memory::MemoryPool*, std::unordered_set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, bool)
# 14 facebook::velox::exec::compileExpressions(std::vector<std::shared_ptr<facebook::velox::core::ITypedExpr const>, std::allocator<std::shared_ptr<facebook::velox::core::ITypedExpr const> > > const&, facebook::velox::core::ExecCtx*, facebook::velox::exec::ExprSet*, bool)
# 15 facebook::velox::exec::ExprSet::ExprSet(std::vector<std::shared_ptr<facebook::velox::core::ITypedExpr const>, std::allocator<std::shared_ptr<facebook::velox::core::ITypedExpr const> > > const&, facebook::velox::core::ExecCtx*, bool)
# 16 std::_MakeUniq<facebook::velox::exec::ExprSet>::__single_object std::make_unique<facebook::velox::exec::ExprSet, std::vector<std::shared_ptr<facebook::velox::core::ITypedExpr const>, std::allocator<std::shared_ptr<facebook::velox::core::ITypedExpr const> > >, facebook::velox::core::ExecCtx*&>(std::vector<std::shared_ptr<facebook::velox::core::ITypedExpr const>, std::allocator<std::shared_ptr<facebook::velox::core::ITypedExpr const> > >&&, facebook::velox::core::ExecCtx*&)
# 17 facebook::velox::exec::makeExprSetFromFlag(std::vector<std::shared_ptr<facebook::velox::core::ITypedExpr const>, std::allocator<std::shared_ptr<facebook::velox::core::ITypedExpr const> > >&&, facebook::velox::core::ExecCtx*)
# 18 facebook::velox::exec::FilterProject::initialize()
# 19 facebook::velox::exec::Driver::initializeOperators()
# 20 facebook::velox::exec::Driver::runInternal(std::shared_ptr<facebook::velox::exec::Driver>&, std::shared_ptr<facebook::velox::exec::BlockingState>&, std::shared_ptr<facebook::velox::RowVector>&)
# 21 facebook::velox::exec::Driver::next(std::shared_ptr<facebook::velox::exec::BlockingState>&)
# 22 facebook::velox::exec::Task::next(folly::SemiFuture<folly::Unit>*)
# 23 gluten::WholeStageResultIterator::next()
# 24 gluten::ResultIterator::getNext()
# 25 gluten::ResultIterator::hasNext()
# 26 Java_org_apache_gluten_vectorized_ColumnarBatchOutIterator_nativeHasNext
# 27 0x00007ff779018507
at org.apache.gluten.vectorized.GeneralOutIterator.hasNext(GeneralOutIterator.java:39)
at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:45)
at org.apache.gluten.utils.IteratorCompleter.hasNext(Iterators.scala:69)
at org.apache.gluten.utils.PayloadCloser.hasNext(Iterators.scala:35)
at org.apache.gluten.utils.PipelineTimeAccumulator.hasNext(Iterators.scala:98)
at scala.collection.Iterator.isEmpty(Iterator.scala:387)
at scala.collection.Iterator.isEmpty$(Iterator.scala:387)
at org.apache.gluten.utils.PipelineTimeAccumulator.isEmpty(Iterators.scala:88)
at org.apache.gluten.execution.VeloxColumnarToRowExec$.toRowIterator(VeloxColumnarToRowExec.scala:119)
at org.apache.gluten.execution.VeloxColumnarToRowExec.$anonfun$doExecuteInternal$1(VeloxColumnarToRowExec.scala:83)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2(RDD.scala:855)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitions$2$adapted(RDD.scala:855)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:136)
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [VL] Add a bad test case when bloom_filter_agg is fallen back while might_contain is not [incubator-gluten]
Posted by "GlutenPerfBot (via GitHub)" <gi...@apache.org>.
GlutenPerfBot commented on PR #5433:
URL: https://github.com/apache/incubator-gluten/pull/5433#issuecomment-2060708410
===== Performance report for TPCH SF2000 with Velox backend, for reference only ====
<table border="1" cellpadding="0" cellspacing="0">
<tbody>
<tr>
<td><font color=#000000>query</font></td>
<td><font color=#000000>log/native_5433_time.csv</font></td>
<td><font color=#000000>log/native_master_04_16_2024_33d183203_time.csv</font></td>
<td><font color=#000000>difference</font></td>
<td><font color=#000000>percentage</font></td>
</tr>
<tr>
<td><font color=#6F9915>q1</font></td>
<td><font color=#6F9915>38.23</font></td>
<td><font color=#6F9915>38.05</font></td>
<td><font color=#6F9915>-0.183</font></td>
<td><font color=#6F9915>99.52%</font></td>
</tr>
<tr>
<td><font color=#F92663>q2</font></td>
<td><font color=#F92663>23.70</font></td>
<td><font color=#F92663>24.39</font></td>
<td><font color=#F92663>0.691</font></td>
<td><font color=#F92663>102.91%</font></td>
</tr>
<tr>
<td><font color=#F92663>q3</font></td>
<td><font color=#F92663>37.27</font></td>
<td><font color=#F92663>37.50</font></td>
<td><font color=#F92663>0.237</font></td>
<td><font color=#F92663>100.64%</font></td>
</tr>
<tr>
<td><font color=#F92663>q4</font></td>
<td><font color=#F92663>37.43</font></td>
<td><font color=#F92663>38.79</font></td>
<td><font color=#F92663>1.366</font></td>
<td><font color=#F92663>103.65%</font></td>
</tr>
<tr>
<td><font color=#6F9915>q5</font></td>
<td><font color=#6F9915>70.88</font></td>
<td><font color=#6F9915>70.62</font></td>
<td><font color=#6F9915>-0.256</font></td>
<td><font color=#6F9915>99.64%</font></td>
</tr>
<tr>
<td><font color=#F92663>q6</font></td>
<td><font color=#F92663>7.23</font></td>
<td><font color=#F92663>9.68</font></td>
<td><font color=#F92663>2.446</font></td>
<td><font color=#F92663>133.81%</font></td>
</tr>
<tr>
<td><font color=#F92663>q7</font></td>
<td><font color=#F92663>83.45</font></td>
<td><font color=#F92663>83.76</font></td>
<td><font color=#F92663>0.312</font></td>
<td><font color=#F92663>100.37%</font></td>
</tr>
<tr>
<td><font color=#F92663>q8</font></td>
<td><font color=#F92663>85.60</font></td>
<td><font color=#F92663>86.47</font></td>
<td><font color=#F92663>0.874</font></td>
<td><font color=#F92663>101.02%</font></td>
</tr>
<tr>
<td><font color=#F92663>q9</font></td>
<td><font color=#F92663>123.56</font></td>
<td><font color=#F92663>124.25</font></td>
<td><font color=#F92663>0.691</font></td>
<td><font color=#F92663>100.56%</font></td>
</tr>
<tr>
<td><font color=#F92663>q10</font></td>
<td><font color=#F92663>45.86</font></td>
<td><font color=#F92663>47.10</font></td>
<td><font color=#F92663>1.248</font></td>
<td><font color=#F92663>102.72%</font></td>
</tr>
<tr>
<td><font color=#F92663>q11</font></td>
<td><font color=#F92663>19.80</font></td>
<td><font color=#F92663>20.01</font></td>
<td><font color=#F92663>0.211</font></td>
<td><font color=#F92663>101.07%</font></td>
</tr>
<tr>
<td><font color=#F92663>q12</font></td>
<td><font color=#F92663>27.65</font></td>
<td><font color=#F92663>30.02</font></td>
<td><font color=#F92663>2.373</font></td>
<td><font color=#F92663>108.58%</font></td>
</tr>
<tr>
<td><font color=#6F9915>q13</font></td>
<td><font color=#6F9915>54.96</font></td>
<td><font color=#6F9915>53.95</font></td>
<td><font color=#6F9915>-1.007</font></td>
<td><font color=#6F9915>98.17%</font></td>
</tr>
<tr>
<td><font color=#6F9915>q14</font></td>
<td><font color=#6F9915>19.35</font></td>
<td><font color=#6F9915>17.11</font></td>
<td><font color=#6F9915>-2.244</font></td>
<td><font color=#6F9915>88.41%</font></td>
</tr>
<tr>
<td><font color=#6F9915>q15</font></td>
<td><font color=#6F9915>30.26</font></td>
<td><font color=#6F9915>29.43</font></td>
<td><font color=#6F9915>-0.834</font></td>
<td><font color=#6F9915>97.24%</font></td>
</tr>
<tr>
<td><font color=#F92663>q16</font></td>
<td><font color=#F92663>13.82</font></td>
<td><font color=#F92663>13.85</font></td>
<td><font color=#F92663>0.032</font></td>
<td><font color=#F92663>100.23%</font></td>
</tr>
<tr>
<td><font color=#F92663>q17</font></td>
<td><font color=#F92663>99.56</font></td>
<td><font color=#F92663>101.18</font></td>
<td><font color=#F92663>1.623</font></td>
<td><font color=#F92663>101.63%</font></td>
</tr>
<tr>
<td><font color=#F92663>q18</font></td>
<td><font color=#F92663>143.81</font></td>
<td><font color=#F92663>144.50</font></td>
<td><font color=#F92663>0.682</font></td>
<td><font color=#F92663>100.47%</font></td>
</tr>
<tr>
<td><font color=#F92663>q19</font></td>
<td><font color=#F92663>16.22</font></td>
<td><font color=#F92663>16.75</font></td>
<td><font color=#F92663>0.523</font></td>
<td><font color=#F92663>103.23%</font></td>
</tr>
<tr>
<td><font color=#F92663>q20</font></td>
<td><font color=#F92663>26.87</font></td>
<td><font color=#F92663>26.88</font></td>
<td><font color=#F92663>0.006</font></td>
<td><font color=#F92663>100.02%</font></td>
</tr>
<tr>
<td><font color=#6F9915>q21</font></td>
<td><font color=#6F9915>289.97</font></td>
<td><font color=#6F9915>287.81</font></td>
<td><font color=#6F9915>-2.166</font></td>
<td><font color=#6F9915>99.25%</font></td>
</tr>
<tr>
<td><font color=#F92663>q22</font></td>
<td><font color=#F92663>16.05</font></td>
<td><font color=#F92663>16.10</font></td>
<td><font color=#F92663>0.041</font></td>
<td><font color=#F92663>100.26%</font></td>
</tr>
<tr>
<td><font color=#F92663>total</font></td>
<td><font color=#F92663>1311.52</font></td>
<td><font color=#F92663>1318.19</font></td>
<td><font color=#F92663>6.667</font></td>
<td><font color=#F92663>100.51%</font></td>
</tr>
</tbody>
</table>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [VL] Add a bad test case when bloom_filter_agg is fallen back while might_contain is not [incubator-gluten]
Posted by "zhztheplayer (via GitHub)" <gi...@apache.org>.
zhztheplayer merged PR #5433:
URL: https://github.com/apache/incubator-gluten/pull/5433
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [VL] Add a bad test case when bloom_filter_agg is fallen back while might_contain is not [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #5433:
URL: https://github.com/apache/incubator-gluten/pull/5433#issuecomment-2060305631
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICEGJ-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
Thanks for opening a pull request!
Could you open an issue for this pull request on Github Issues?
https://github.com/apache/incubator-gluten/issues
Then could you also rename ***commit message*** and ***pull request title*** in the following format?
[GLUTEN-${ISSUES_ID}][COMPONENT]feat/fix: ${detailed message}
See also:
* [Other pull requests](https://github.com/apache/incubator-gluten/pulls/)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [VL] Add a bad test case when bloom_filter_agg is fallen back while might_contain is not [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #5433:
URL: https://github.com/apache/incubator-gluten/pull/5433#issuecomment-2060305824
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [VL] Add a bad test case when bloom_filter_agg is fallen back while might_contain is not [incubator-gluten]
Posted by "zhztheplayer (via GitHub)" <gi...@apache.org>.
zhztheplayer commented on code in PR #5433:
URL: https://github.com/apache/incubator-gluten/pull/5433#discussion_r1568187545
##########
gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformHintRule.scala:
##########
@@ -324,7 +324,7 @@ case class FallbackBloomFilterAggIfNeeded() extends Rule[SparkPlan] {
case a: org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec =>
tagNotTransformableRecursive(a.executedPlan)
case _ =>
- p.children.map(tagNotTransformableRecursive)
+ p.children.foreach(tagNotTransformableRecursive)
Review Comment:
just a nit change
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [VL] Add a bad test case when bloom_filter_agg is fallen back while might_contain is not [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #5433:
URL: https://github.com/apache/incubator-gluten/pull/5433#issuecomment-2060311948
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [VL] Add a bad test case when bloom_filter_agg is fallen back while might_contain is not [incubator-gluten]
Posted by "zhztheplayer (via GitHub)" <gi...@apache.org>.
zhztheplayer commented on code in PR #5433:
URL: https://github.com/apache/incubator-gluten/pull/5433#discussion_r1572009913
##########
gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala:
##########
@@ -113,4 +113,37 @@ class GlutenBloomFilterAggregateQuerySuite
}
}
}
+
+ testGluten("Test bloom_filter_agg fallback with might_contain offloaded") {
+ val table = "bloom_filter_test"
+ val numEstimatedItems = 5000000L
+ val numBits = GlutenConfig.getConf.veloxBloomFilterMaxNumBits
+ val sqlString = s"""
+ |SELECT col positive_membership_test
+ |FROM $table
+ |WHERE might_contain(
+ | (SELECT bloom_filter_agg(col,
+ | cast($numEstimatedItems as long),
+ | cast($numBits as long))
+ | FROM $table), col)
+ """.stripMargin
+
+ withTempView(table) {
+ (Seq(Long.MinValue, 0, Long.MaxValue) ++ (1L to 200000L))
+ .toDF("col")
+ .createOrReplaceTempView(table)
+ withSQLConf(
+ GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key -> "false"
Review Comment:
> If this is the only case that triggers bloom_filter_agg fallback?
Probably there are still some cases making agg fallback, e.g., validation failures by other agg functions. Since the agg and might_contain are not in the same query/sub-query, plus taking AQE on/off and other validation/transformation rules into account, doing such co-fallback can be a very dirty work. Let's continue with the new approach introduced in https://github.com/apache/incubator-gluten/pull/5435 to let vanilla Spark be able to run Velox's bloom filter then we can thoroughly solve all the issues related to bloom filter mismatch including these fallback problems.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [VL] Add a bad test case when bloom_filter_agg is fallen back while might_contain is not [incubator-gluten]
Posted by "zhli1142015 (via GitHub)" <gi...@apache.org>.
zhli1142015 commented on code in PR #5433:
URL: https://github.com/apache/incubator-gluten/pull/5433#discussion_r1569728838
##########
gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenBloomFilterAggregateQuerySuite.scala:
##########
@@ -113,4 +113,37 @@ class GlutenBloomFilterAggregateQuerySuite
}
}
}
+
+ testGluten("Test bloom_filter_agg fallback with might_contain offloaded") {
+ val table = "bloom_filter_test"
+ val numEstimatedItems = 5000000L
+ val numBits = GlutenConfig.getConf.veloxBloomFilterMaxNumBits
+ val sqlString = s"""
+ |SELECT col positive_membership_test
+ |FROM $table
+ |WHERE might_contain(
+ | (SELECT bloom_filter_agg(col,
+ | cast($numEstimatedItems as long),
+ | cast($numBits as long))
+ | FROM $table), col)
+ """.stripMargin
+
+ withTempView(table) {
+ (Seq(Long.MinValue, 0, Long.MaxValue) ++ (1L to 200000L))
+ .toDF("col")
+ .createOrReplaceTempView(table)
+ withSQLConf(
+ GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key -> "false"
Review Comment:
If this is the only case that triggers bloom_filter_agg fallback?
If yes we may change the flag for native bloom filter as below:
`def enableNativeBloomFilter: Boolean = conf.getConf(COLUMNAR_NATIVE_BLOOMFILTER_ENABLED) && conf.getConf(COLUMNAR_HASHAGG_ENABLED)`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org