You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ke...@apache.org on 2024/09/26 06:10:02 UTC
(incubator-gluten) branch main updated: [GLUTEN-6887][VL] Daily Update Velox Version (2024_09_26) (#7355)
This is an automated email from the ASF dual-hosted git repository.
kejia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 624752de22 [GLUTEN-6887][VL] Daily Update Velox Version (2024_09_26) (#7355)
624752de22 is described below
commit 624752de220672b7c1ecfe882419fe8bc79156b3
Author: JiaKe <ke...@intel.com>
AuthorDate: Thu Sep 26 14:09:57 2024 +0800
[GLUTEN-6887][VL] Daily Update Velox Version (2024_09_26) (#7355)
---
cpp/velox/memory/VeloxMemoryManager.cc | 9 ++++++---
ep/build-velox/src/get_velox.sh | 2 +-
.../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 1 -
.../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 1 -
.../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 1 -
.../scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala | 6 ------
6 files changed, 7 insertions(+), 13 deletions(-)
diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc
index c3845d1f0d..93fc4f6f52 100644
--- a/cpp/velox/memory/VeloxMemoryManager.cc
+++ b/cpp/velox/memory/VeloxMemoryManager.cc
@@ -210,6 +210,11 @@ VeloxMemoryManager::VeloxMemoryManager(std::unique_ptr<AllocationListener> liste
listenableAlloc_ = std::make_unique<ListenableMemoryAllocator>(defaultMemoryAllocator().get(), blockListener_.get());
arrowPool_ = std::make_unique<ArrowMemoryPool>(listenableAlloc_.get());
+ std::unordered_map<std::string, std::string> extraArbitratorConfigs;
+ extraArbitratorConfigs["memory-pool-initial-capacity"] = folly::to<std::string>(memInitCapacity) + "B";
+ extraArbitratorConfigs["memory-pool-transfer-capacity"] = folly::to<std::string>(reservationBlockSize) + "B";
+ extraArbitratorConfigs["memory-reclaim-max-wait-time"] = folly::to<std::string>(0) + "ms";
+
ArbitratorFactoryRegister afr(listener_.get());
velox::memory::MemoryManagerOptions mmOptions{
.alignment = velox::memory::MemoryAllocator::kMaxAlignment,
@@ -219,9 +224,7 @@ VeloxMemoryManager::VeloxMemoryManager(std::unique_ptr<AllocationListener> liste
.coreOnAllocationFailureEnabled = false,
.allocatorCapacity = velox::memory::kMaxMemory,
.arbitratorKind = afr.getKind(),
- .memoryPoolInitCapacity = memInitCapacity,
- .memoryPoolTransferCapacity = reservationBlockSize,
- .memoryReclaimWaitMs = 0};
+ .extraArbitratorConfigs = extraArbitratorConfigs};
veloxMemoryManager_ = std::make_unique<velox::memory::MemoryManager>(mmOptions);
veloxAggregatePool_ = veloxMemoryManager_->addRootPool(
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
index 2e33713abc..010113cc66 100755
--- a/ep/build-velox/src/get_velox.sh
+++ b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
set -exu
VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2024_09_25
+VELOX_BRANCH=2024_09_26
VELOX_HOME=""
OS=`uname -s`
diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 43c1690890..337aa5025f 100644
--- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -224,7 +224,6 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenDecimalPrecisionSuite]
enableSuite[GlutenStringFunctionsSuite]
enableSuite[GlutenRegexpExpressionsSuite]
- .excludeByPrefix("LIKE Pattern ESCAPE")
enableSuite[GlutenNullExpressionsSuite]
enableSuite[GlutenPredicateSuite]
enableSuite[GlutenMathExpressionsSuite]
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 08d133034f..caa91891cf 100644
--- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -162,7 +162,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("random")
.exclude("SPARK-9127 codegen with long seed")
enableSuite[GlutenRegexpExpressionsSuite]
- .excludeByPrefix("LIKE Pattern ESCAPE")
enableSuite[GlutenSortShuffleSuite]
enableSuite[GlutenSortOrderExpressionsSuite]
enableSuite[GlutenStringExpressionsSuite]
diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 3fbac698b0..963fb79a35 100644
--- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -140,7 +140,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("random")
.exclude("SPARK-9127 codegen with long seed")
enableSuite[GlutenRegexpExpressionsSuite]
- .excludeByPrefix("LIKE Pattern ESCAPE")
enableSuite[GlutenSortShuffleSuite]
enableSuite[GlutenSortOrderExpressionsSuite]
enableSuite[GlutenStringExpressionsSuite]
diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index a3bdbb1371..ed4939595b 100644
--- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -143,7 +143,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("random")
.exclude("SPARK-9127 codegen with long seed")
enableSuite[GlutenRegexpExpressionsSuite]
- .excludeByPrefix("LIKE Pattern ESCAPE")
enableSuite[GlutenSortShuffleSuite]
enableSuite[GlutenSortOrderExpressionsSuite]
enableSuite[GlutenStringExpressionsSuite]
@@ -884,11 +883,6 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenInnerJoinSuiteForceShjOff]
enableSuite[GlutenOuterJoinSuiteForceShjOn]
enableSuite[GlutenOuterJoinSuiteForceShjOff]
- // Caused by Velox SMJ result mismatches with Spark.
- .exclude("basic right outer join using SortMergeJoin (whole-stage-codegen off)")
- .exclude("basic right outer join using SortMergeJoin (whole-stage-codegen on)")
- .exclude("right outer join with unique keys using SortMergeJoin (whole-stage-codegen off)")
- .exclude("right outer join with unique keys using SortMergeJoin (whole-stage-codegen on)")
enableSuite[FallbackStrategiesSuite]
enableSuite[GlutenBroadcastExchangeSuite]
enableSuite[GlutenLocalBroadcastExchangeSuite]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org