You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ho...@apache.org on 2024/07/01 03:36:25 UTC
(incubator-gluten) branch main updated: [CORE] Execution runtime / native memory manager refactor (#6243)
This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ae2d12557 [CORE] Execution runtime / native memory manager refactor (#6243)
ae2d12557 is described below
commit ae2d1255739a0a4c37f35d3a938c2af30849aeae
Author: Hongze Zhang <ho...@intel.com>
AuthorDate: Mon Jul 1 11:36:18 2024 +0800
[CORE] Execution runtime / native memory manager refactor (#6243)
---
.github/workflows/velox_docker.yml | 27 +-
.../memory/alloc/CHNativeMemoryAllocators.java | 13 +-
.../spark/shuffle/CHColumnarShuffleWriter.scala | 8 +-
.../spark/memory/TestTaskMemoryManagerSuite.java | 3 +-
.../apache/gluten/utils/VeloxBatchAppender.java | 12 +-
.../gluten/utils/VeloxBatchAppenderJniWrapper.java | 5 +-
.../org/apache/gluten/utils/VeloxBloomFilter.java | 8 +-
.../gluten/utils/VeloxBloomFilterJniWrapper.java | 5 +-
.../backendsapi/velox/VeloxSparkPlanExecApi.scala | 26 +-
.../backendsapi/velox/VeloxTransformerApi.scala | 4 +-
.../gluten/execution/RowToVeloxColumnarExec.scala | 16 +-
.../gluten/execution/VeloxColumnarToRowExec.scala | 27 +-
.../org/apache/gluten/utils/DatasourceUtil.scala | 9 +-
.../spark/sql/execution/BroadcastUtils.scala | 12 +-
.../execution/ColumnarCachedBatchSerializer.scala | 32 +-
.../datasources/velox/VeloxBlockStripes.java | 3 +-
.../velox/VeloxFormatWriterInjects.scala | 22 +-
cpp/core/compute/Runtime.cc | 7 +-
cpp/core/compute/Runtime.h | 52 ++-
cpp/core/jni/JniCommon.cc | 2 +-
cpp/core/jni/JniCommon.h | 3 +-
cpp/core/jni/JniError.cc | 7 +
cpp/core/jni/JniError.h | 13 +-
cpp/core/jni/JniWrapper.cc | 390 ++++++++-------------
cpp/core/tests/CMakeLists.txt | 1 +
cpp/core/tests/ObjectStoreTest.cc | 116 ++++++
cpp/core/utils/ObjectStore.cc | 16 +-
cpp/core/utils/ObjectStore.h | 64 +++-
cpp/core/utils/ResourceMap.h | 40 ++-
cpp/velox/benchmarks/GenericBenchmark.cc | 104 +++---
cpp/velox/benchmarks/ParquetWriteBenchmark.cc | 2 +-
cpp/velox/compute/VeloxBackend.cc | 6 +-
cpp/velox/compute/VeloxRuntime.cc | 73 ++--
cpp/velox/compute/VeloxRuntime.h | 59 +---
cpp/velox/compute/WholeStageResultIterator.cc | 7 +-
cpp/velox/jni/VeloxJniWrapper.cc | 27 +-
cpp/velox/memory/VeloxMemoryManager.cc | 21 +-
cpp/velox/memory/VeloxMemoryManager.h | 10 +-
cpp/velox/tests/MemoryManagerTest.cc | 2 +-
cpp/velox/tests/RuntimeTest.cc | 74 ++--
ep/build-velox/src/build_velox.sh | 2 +-
.../CHCelebornHashBasedColumnarShuffleWriter.scala | 12 +-
.../VeloxCelebornColumnarBatchSerializer.scala | 13 +-
...loxCelebornHashBasedColumnarShuffleWriter.scala | 56 ++-
.../gluten/memory/memtarget/MemoryTargets.java | 5 +-
.../apache/gluten/memory/memtarget/Spiller.java | 9 +-
.../apache/gluten/memory/memtarget/Spillers.java | 39 ++-
.../gluten/memory/memtarget/TreeMemoryTarget.java | 5 +-
.../gluten/memory/memtarget/TreeMemoryTargets.java | 41 +--
.../memtarget/spark/RegularMemoryConsumer.java | 12 +-
.../memory/memtarget/spark/TreeMemoryConsumer.java | 15 +-
.../memtarget/spark/TreeMemoryConsumers.java | 8 +-
.../org/apache/spark/memory/SparkMemoryUtil.scala | 4 +
.../memtarget/spark/TreeMemoryConsumerTest.java | 9 +-
.../columnarbatch/ColumnarBatchJniWrapper.java | 22 +-
.../gluten/columnarbatch/ColumnarBatches.java | 50 +--
.../gluten/columnarbatch/IndicatorVector.java | 129 +------
...dicatorVector.java => IndicatorVectorBase.java} | 65 ++--
.../gluten/datasource/DatasourceJniWrapper.java | 16 +-
.../org/apache/gluten/exec/RuntimeJniWrapper.java | 12 +-
.../gluten/memory/alloc/NativeMemoryAllocator.java | 49 ---
.../memory/alloc/NativeMemoryAllocators.java | 54 ---
.../memory/arrow/alloc/ArrowBufferAllocators.java | 6 +-
.../ManagedReservationListener.java | 2 +-
.../{nmm => listener}/ReservationListener.java | 7 +-
.../memory/listener/ReservationListeners.java | 77 ++++
.../gluten/memory/nmm/NativeMemoryManager.java | 124 -------
.../gluten/memory/nmm/NativeMemoryManagers.java | 167 ---------
.../gluten/vectorized/ColumnarBatchInIterator.java | 5 +-
.../vectorized/ColumnarBatchOutIterator.java | 10 +-
.../ColumnarBatchSerializerJniWrapper.java | 11 +-
.../vectorized/NativeColumnarToRowJniWrapper.java | 9 +-
.../gluten/vectorized/NativePlanEvaluator.java | 55 +--
.../vectorized/NativeRowToColumnarJniWrapper.java | 7 +-
.../gluten/vectorized/PlanEvaluatorJniWrapper.java | 7 +-
.../gluten/vectorized/ShuffleReaderJniWrapper.java | 6 +-
.../gluten/vectorized/ShuffleWriterJniWrapper.java | 10 +-
.../scala/org/apache/gluten/exec/Runtime.scala | 116 ++++--
.../scala/org/apache/gluten/exec/Runtimes.scala | 11 +-
.../vectorized/ColumnarBatchSerializer.scala | 17 +-
.../spark/shuffle/ColumnarShuffleWriter.scala | 47 +--
.../sql/execution/ColumnarBuildSideRelation.scala | 26 +-
.../spark/sql/execution/utils/ExecUtil.scala | 17 +-
.../writer/VeloxUniffleColumnarShuffleWriter.java | 58 ++-
84 files changed, 1172 insertions(+), 1578 deletions(-)
diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml
index fd937f6c1..098b2a2d5 100644
--- a/.github/workflows/velox_docker.yml
+++ b/.github/workflows/velox_docker.yml
@@ -292,7 +292,7 @@ jobs:
- name: TPC-DS SF30.0 Parquet local spark3.2 Q67/Q95 low memory, memory isolation off
run: |
cd tools/gluten-it \
- && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh parameterized \
+ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
--local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q67,q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
--skip-data-gen -m=OffHeapExecutionMemory \
-d=ISOLATION:OFF,spark.gluten.memory.isolation=false \
@@ -304,7 +304,7 @@ jobs:
- name: TPC-DS SF30.0 Parquet local spark3.2 Q67 low memory, memory isolation on
run: |
cd tools/gluten-it \
- && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh parameterized \
+ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
--local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q67 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
--skip-data-gen -m=OffHeapExecutionMemory \
-d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \
@@ -315,7 +315,7 @@ jobs:
- name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q95 low memory, memory isolation on
run: |
cd tools/gluten-it \
- && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh parameterized \
+ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
--local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
--skip-data-gen -m=OffHeapExecutionMemory \
-d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \
@@ -326,10 +326,21 @@ jobs:
- name: TPC-DS SF30.0 Parquet local spark3.2 Q23A/Q23B low memory
run: |
cd tools/gluten-it \
- && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh parameterized \
+ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
--local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q23a,q23b -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
--skip-data-gen -m=OffHeapExecutionMemory \
-d=ISOLATION:OFF,spark.gluten.memory.isolation=false \
+ -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \
+ -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \
+ -d=FLUSH_MODE:ABANDONED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \
+ -d=FLUSH_MODE:FLUSHED,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=0.05,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=0.1,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0
+ - name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q23A/Q23B low memory, memory isolation on
+ if: false # Disabled as error https://gist.github.com/zhztheplayer/abd5e83ccdc48730678ae7ebae479fcc
+ run: |
+ cd tools/gluten-it \
+ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
+ --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q23a,q23b -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
+ --skip-data-gen -m=OffHeapExecutionMemory \
-d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \
-d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \
-d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \
@@ -338,7 +349,7 @@ jobs:
- name: (To be fixed) TPC-DS SF30.0 Parquet local spark3.2 Q97 low memory # The case currently causes crash with "free: invalid size".
run: |
cd tools/gluten-it \
- && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh parameterized \
+ && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
--local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q97 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
--skip-data-gen -m=OffHeapExecutionMemory \
-d=ISOLATION:OFF,spark.gluten.memory.isolation=false \
@@ -455,7 +466,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- spark: ["spark-3.2"]
+ spark: [ "spark-3.2" ]
runs-on: ubuntu-20.04
container: centos:8
steps:
@@ -520,8 +531,8 @@ jobs:
strategy:
fail-fast: false
matrix:
- spark: ["spark-3.2"]
- celeborn: ["celeborn-0.4.1", "celeborn-0.3.2-incubating"]
+ spark: [ "spark-3.2" ]
+ celeborn: [ "celeborn-0.4.1", "celeborn-0.3.2-incubating" ]
runs-on: ubuntu-20.04
container: ubuntu:22.04
steps:
diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocators.java b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocators.java
index 01fb7e3e2..0f30972fc 100644
--- a/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocators.java
+++ b/backends-clickhouse/src/main/java/org/apache/gluten/memory/alloc/CHNativeMemoryAllocators.java
@@ -19,13 +19,12 @@ package org.apache.gluten.memory.alloc;
import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
import org.apache.gluten.memory.memtarget.MemoryTargets;
import org.apache.gluten.memory.memtarget.Spiller;
+import org.apache.gluten.memory.memtarget.Spillers;
import org.apache.spark.memory.TaskMemoryManager;
import org.apache.spark.util.TaskResources;
-import java.util.Arrays;
import java.util.Collections;
-import java.util.List;
/**
* Built-in toolkit for managing native memory allocations. To use the facility, one should import
@@ -46,12 +45,12 @@ public abstract class CHNativeMemoryAllocators {
private static CHNativeMemoryAllocatorManager createNativeMemoryAllocatorManager(
String name,
TaskMemoryManager taskMemoryManager,
- List<Spiller> spillers,
+ Spiller spiller,
SimpleMemoryUsageRecorder usage) {
CHManagedCHReservationListener rl =
new CHManagedCHReservationListener(
- MemoryTargets.newConsumer(taskMemoryManager, name, spillers, Collections.emptyMap()),
+ MemoryTargets.newConsumer(taskMemoryManager, name, spiller, Collections.emptyMap()),
usage);
return new CHNativeMemoryAllocatorManagerImpl(CHNativeMemoryAllocator.createListenable(rl));
}
@@ -67,7 +66,7 @@ public abstract class CHNativeMemoryAllocators {
createNativeMemoryAllocatorManager(
"ContextInstance",
TaskResources.getLocalTaskContext().taskMemoryManager(),
- Collections.emptyList(),
+ Spillers.NOOP,
TaskResources.getSharedUsage());
TaskResources.addResource(id, manager);
}
@@ -78,7 +77,7 @@ public abstract class CHNativeMemoryAllocators {
return CHNativeMemoryAllocator.getDefaultForUT();
}
- public static CHNativeMemoryAllocator createSpillable(String name, Spiller... spillers) {
+ public static CHNativeMemoryAllocator createSpillable(String name, Spiller spiller) {
if (!TaskResources.inSparkTask()) {
throw new IllegalStateException("spiller must be used in a Spark task");
}
@@ -87,7 +86,7 @@ public abstract class CHNativeMemoryAllocators {
createNativeMemoryAllocatorManager(
name,
TaskResources.getLocalTaskContext().taskMemoryManager(),
- Arrays.asList(spillers),
+ spiller,
TaskResources.getSharedUsage());
TaskResources.addAnonymousResource(manager);
// force add memory consumer to task memory manager, will release by inactivate
diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala
index 4a1adbec7..c113f8d4d 100644
--- a/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala
+++ b/backends-clickhouse/src/main/scala/org/apache/spark/shuffle/CHColumnarShuffleWriter.scala
@@ -29,7 +29,6 @@ import org.apache.spark.sql.vectorized.ColumnarBatch
import org.apache.spark.util.{SparkDirectoryUtil, Utils}
import java.io.IOException
-import java.util
import java.util.{Locale, UUID}
class CHColumnarShuffleWriter[K, V](
@@ -122,7 +121,10 @@ class CHColumnarShuffleWriter[K, V](
CHNativeMemoryAllocators.createSpillable(
"ShuffleWriter",
new Spiller() {
- override def spill(self: MemoryTarget, size: Long): Long = {
+ override def spill(self: MemoryTarget, phase: Spiller.Phase, size: Long): Long = {
+ if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) {
+ return 0L;
+ }
if (nativeSplitter == 0) {
throw new IllegalStateException(
"Fatal: spill() called before a shuffle writer " +
@@ -134,8 +136,6 @@ class CHColumnarShuffleWriter[K, V](
logError(s"Gluten shuffle writer: Spilled $spilled / $size bytes of data")
spilled
}
-
- override def applicablePhases(): util.Set[Spiller.Phase] = Spillers.PHASE_SET_SPILL_ONLY
}
)
}
diff --git a/backends-clickhouse/src/test/java/org/apache/spark/memory/TestTaskMemoryManagerSuite.java b/backends-clickhouse/src/test/java/org/apache/spark/memory/TestTaskMemoryManagerSuite.java
index b575de403..905ffacde 100644
--- a/backends-clickhouse/src/test/java/org/apache/spark/memory/TestTaskMemoryManagerSuite.java
+++ b/backends-clickhouse/src/test/java/org/apache/spark/memory/TestTaskMemoryManagerSuite.java
@@ -21,6 +21,7 @@ import org.apache.gluten.memory.alloc.CHManagedCHReservationListener;
import org.apache.gluten.memory.alloc.CHNativeMemoryAllocator;
import org.apache.gluten.memory.alloc.CHNativeMemoryAllocatorManagerImpl;
import org.apache.gluten.memory.memtarget.MemoryTargets;
+import org.apache.gluten.memory.memtarget.Spillers;
import org.apache.spark.SparkConf;
import org.apache.spark.internal.config.package$;
@@ -52,7 +53,7 @@ public class TestTaskMemoryManagerSuite {
listener =
new CHManagedCHReservationListener(
MemoryTargets.newConsumer(
- taskMemoryManager, "test", Collections.emptyList(), Collections.emptyMap()),
+ taskMemoryManager, "test", Spillers.NOOP, Collections.emptyMap()),
new SimpleMemoryUsageRecorder());
manager = new CHNativeMemoryAllocatorManagerImpl(new CHNativeMemoryAllocator(-1L, listener));
diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java
index 1bf34b5ce..32b228947 100644
--- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java
+++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppender.java
@@ -18,8 +18,6 @@ package org.apache.gluten.utils;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.Runtimes;
-import org.apache.gluten.memory.nmm.NativeMemoryManager;
-import org.apache.gluten.memory.nmm.NativeMemoryManagers;
import org.apache.gluten.vectorized.ColumnarBatchInIterator;
import org.apache.gluten.vectorized.ColumnarBatchOutIterator;
@@ -30,12 +28,10 @@ import java.util.Iterator;
public final class VeloxBatchAppender {
public static ColumnarBatchOutIterator create(
int minOutputBatchSize, Iterator<ColumnarBatch> in) {
- final Runtime runtime = Runtimes.contextInstance();
- final NativeMemoryManager nmm = NativeMemoryManagers.contextInstance("VeloxBatchAppender");
+ final Runtime runtime = Runtimes.contextInstance("VeloxBatchAppender");
long outHandle =
- VeloxBatchAppenderJniWrapper.forRuntime(runtime)
- .create(
- nmm.getNativeInstanceHandle(), minOutputBatchSize, new ColumnarBatchInIterator(in));
- return new ColumnarBatchOutIterator(runtime, outHandle, nmm);
+ VeloxBatchAppenderJniWrapper.create(runtime)
+ .create(minOutputBatchSize, new ColumnarBatchInIterator(in));
+ return new ColumnarBatchOutIterator(runtime, outHandle);
}
}
diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java
index 9e2531951..231e65553 100644
--- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java
+++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBatchAppenderJniWrapper.java
@@ -27,7 +27,7 @@ public class VeloxBatchAppenderJniWrapper implements RuntimeAware {
this.runtime = runtime;
}
- public static VeloxBatchAppenderJniWrapper forRuntime(Runtime runtime) {
+ public static VeloxBatchAppenderJniWrapper create(Runtime runtime) {
return new VeloxBatchAppenderJniWrapper(runtime);
}
@@ -36,6 +36,5 @@ public class VeloxBatchAppenderJniWrapper implements RuntimeAware {
return runtime.getHandle();
}
- public native long create(
- long memoryManagerHandle, int minOutputBatchSize, ColumnarBatchInIterator itr);
+ public native long create(int minOutputBatchSize, ColumnarBatchInIterator itr);
}
diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java
index 13ba8e011..f23426d7d 100644
--- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java
+++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilter.java
@@ -16,6 +16,8 @@
*/
package org.apache.gluten.utils;
+import org.apache.gluten.exec.Runtimes;
+
import org.apache.commons.io.IOUtils;
import org.apache.spark.util.sketch.BloomFilter;
import org.apache.spark.util.sketch.IncompatibleMergeException;
@@ -27,17 +29,15 @@ import java.io.InputStream;
import java.io.OutputStream;
public class VeloxBloomFilter extends BloomFilter {
-
- private final VeloxBloomFilterJniWrapper jni;
+ private final VeloxBloomFilterJniWrapper jni =
+ VeloxBloomFilterJniWrapper.create(Runtimes.contextInstance("VeloxBloomFilter"));
private final long handle;
private VeloxBloomFilter(byte[] data) {
- jni = VeloxBloomFilterJniWrapper.create();
handle = jni.init(data);
}
private VeloxBloomFilter(int capacity) {
- jni = VeloxBloomFilterJniWrapper.create();
handle = jni.empty(capacity);
}
diff --git a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java
index 94f8e17bc..3ddfd2c02 100644
--- a/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java
+++ b/backends-velox/src/main/java/org/apache/gluten/utils/VeloxBloomFilterJniWrapper.java
@@ -18,7 +18,6 @@ package org.apache.gluten.utils;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.RuntimeAware;
-import org.apache.gluten.exec.Runtimes;
public class VeloxBloomFilterJniWrapper implements RuntimeAware {
private final Runtime runtime;
@@ -27,8 +26,8 @@ public class VeloxBloomFilterJniWrapper implements RuntimeAware {
this.runtime = runtime;
}
- public static VeloxBloomFilterJniWrapper create() {
- return new VeloxBloomFilterJniWrapper(Runtimes.contextInstance());
+ public static VeloxBloomFilterJniWrapper create(Runtime runtime) {
+ return new VeloxBloomFilterJniWrapper(runtime);
}
@Override
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index ed69a5893..582bf997f 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -110,8 +110,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
GenericExpressionTransformer(condFuncName, Seq(left), condExpr),
right,
left,
- newExpr
- )
+ newExpr)
}
/** Transform Uuid to Substrait. */
@@ -488,8 +487,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
left,
right,
isSkewJoin,
- projectList
- )
+ projectList)
}
override def genCartesianProductExecTransformer(
left: SparkPlan,
@@ -498,8 +496,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
CartesianProductExecTransformer(
ColumnarCartesianProductBridge(left),
ColumnarCartesianProductBridge(right),
- condition
- )
+ condition)
}
override def genBroadcastNestedLoopJoinExecTransformer(
@@ -508,13 +505,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
buildSide: BuildSide,
joinType: JoinType,
condition: Option[Expression]): BroadcastNestedLoopJoinExecTransformer =
- VeloxBroadcastNestedLoopJoinExecTransformer(
- left,
- right,
- buildSide,
- joinType,
- condition
- )
+ VeloxBroadcastNestedLoopJoinExecTransformer(left, right, buildSide, joinType, condition)
override def genHashExpressionTransformer(
substraitExprName: String,
@@ -795,10 +786,8 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
*
* @return
*/
- override def genExtendedOptimizers(): List[SparkSession => Rule[LogicalPlan]] = List(
- CollectRewriteRule.apply,
- HLLRewriteRule.apply
- )
+ override def genExtendedOptimizers(): List[SparkSession => Rule[LogicalPlan]] =
+ List(CollectRewriteRule.apply, HLLRewriteRule.apply)
/**
* Generate extended columnar pre-rules, in the validation phase.
@@ -879,8 +868,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
requiredChildOutput: Seq[Attribute],
outer: Boolean,
generatorOutput: Seq[Attribute],
- child: SparkPlan
- ): GenerateExecTransformerBase = {
+ child: SparkPlan): GenerateExecTransformerBase = {
GenerateExecTransformer(generator, requiredChildOutput, outer, generatorOutput, child)
}
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
index ac24b53af..4cbde635e 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
@@ -17,6 +17,7 @@
package org.apache.gluten.backendsapi.velox
import org.apache.gluten.backendsapi.TransformerApi
+import org.apache.gluten.exec.Runtimes
import org.apache.gluten.expression.ConverterUtils
import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode}
import org.apache.gluten.utils.InputPartitionsUtil
@@ -83,7 +84,8 @@ class VeloxTransformerApi extends TransformerApi with Logging {
override def getNativePlanString(substraitPlan: Array[Byte], details: Boolean): String = {
TaskResources.runUnsafe {
- val jniWrapper = PlanEvaluatorJniWrapper.create()
+ val jniWrapper = PlanEvaluatorJniWrapper.create(
+ Runtimes.contextInstance("VeloxTransformerApi#getNativePlanString"))
jniWrapper.nativePlanString(substraitPlan, details)
}
}
diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
index d694f15fa..289df1a6e 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala
@@ -21,7 +21,6 @@ import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.exception.GlutenException
import org.apache.gluten.exec.Runtimes
import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.ArrowAbiUtil
import org.apache.gluten.utils.iterator.Iterators
import org.apache.gluten.vectorized._
@@ -71,8 +70,7 @@ case class RowToVeloxColumnarExec(child: SparkPlan) extends RowToColumnarExecBas
numInputRows,
numOutputBatches,
convertTime,
- numRows
- )
+ numRows)
}
}
@@ -97,9 +95,7 @@ case class RowToVeloxColumnarExec(child: SparkPlan) extends RowToColumnarExecBas
numInputRows,
numOutputBatches,
convertTime,
- numRows
- )
- )
+ numRows))
}
// For spark 3.2.
@@ -121,16 +117,16 @@ object RowToVeloxColumnarExec {
val arrowSchema =
SparkArrowUtil.toArrowSchema(schema, SQLConf.get.sessionLocalTimeZone)
- val jniWrapper = NativeRowToColumnarJniWrapper.create()
+ val runtime = Runtimes.contextInstance("RowToColumnar")
+ val jniWrapper = NativeRowToColumnarJniWrapper.create(runtime)
val arrowAllocator = ArrowBufferAllocators.contextInstance()
- val memoryManager = NativeMemoryManagers.contextInstance("RowToColumnar")
val cSchema = ArrowSchema.allocateNew(arrowAllocator)
val factory = UnsafeProjection
val converter = factory.create(schema)
val r2cHandle =
try {
ArrowAbiUtil.exportSchema(arrowAllocator, arrowSchema, cSchema)
- jniWrapper.init(cSchema.memoryAddress(), memoryManager.getNativeInstanceHandle)
+ jniWrapper.init(cSchema.memoryAddress())
} finally {
cSchema.close()
}
@@ -220,7 +216,7 @@ object RowToVeloxColumnarExec {
try {
val handle = jniWrapper
.nativeConvertRowToColumnar(r2cHandle, rowLength.toArray, arrowBuf.memoryAddress())
- val cb = ColumnarBatches.create(Runtimes.contextInstance(), handle)
+ val cb = ColumnarBatches.create(runtime, handle)
convertTime += System.currentTimeMillis() - startNative
cb
} finally {
diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala
index 0d6714d3a..1a5425520 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/execution/VeloxColumnarToRowExec.scala
@@ -18,8 +18,8 @@ package org.apache.gluten.execution
import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.exception.GlutenNotSupportException
+import org.apache.gluten.exec.Runtimes
import org.apache.gluten.extension.ValidationResult
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.iterator.Iterators
import org.apache.gluten.vectorized.NativeColumnarToRowJniWrapper
@@ -75,13 +75,8 @@ case class VeloxColumnarToRowExec(child: SparkPlan) extends ColumnarToRowExecBas
val convertTime = longMetric("convertTime")
child.executeColumnar().mapPartitions {
it =>
- VeloxColumnarToRowExec.toRowIterator(
- it,
- output,
- numOutputRows,
- numInputBatches,
- convertTime
- )
+ VeloxColumnarToRowExec
+ .toRowIterator(it, output, numOutputRows, numInputBatches, convertTime)
}
}
@@ -96,13 +91,7 @@ case class VeloxColumnarToRowExec(child: SparkPlan) extends ColumnarToRowExecBas
sparkContext,
mode,
relation,
- VeloxColumnarToRowExec.toRowIterator(
- _,
- output,
- numOutputRows,
- numInputBatches,
- convertTime
- ))
+ VeloxColumnarToRowExec.toRowIterator(_, output, numOutputRows, numInputBatches, convertTime))
}
protected def withNewChildInternal(newChild: SparkPlan): VeloxColumnarToRowExec =
@@ -120,10 +109,10 @@ object VeloxColumnarToRowExec {
return Iterator.empty
}
+ val runtime = Runtimes.contextInstance("ColumnarToRow")
// TODO:: pass the jni jniWrapper and arrowSchema and serializeSchema method by broadcast
- val jniWrapper = NativeColumnarToRowJniWrapper.create()
- val c2rId = jniWrapper.nativeColumnarToRowInit(
- NativeMemoryManagers.contextInstance("ColumnarToRow").getNativeInstanceHandle)
+ val jniWrapper = NativeColumnarToRowJniWrapper.create(runtime)
+ val c2rId = jniWrapper.nativeColumnarToRowInit()
val res: Iterator[Iterator[InternalRow]] = new Iterator[Iterator[InternalRow]] {
@@ -159,7 +148,7 @@ object VeloxColumnarToRowExec {
val beforeConvert = System.currentTimeMillis()
val batchHandle = ColumnarBatches.getNativeHandle(batch)
val info =
- jniWrapper.nativeColumnarToRowConvert(batchHandle, c2rId)
+ jniWrapper.nativeColumnarToRowConvert(c2rId, batchHandle)
convertTime += (System.currentTimeMillis() - beforeConvert)
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala
index 6150507b4..3d7725655 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/utils/DatasourceUtil.scala
@@ -17,8 +17,8 @@
package org.apache.gluten.utils
import org.apache.gluten.datasource.DatasourceJniWrapper
+import org.apache.gluten.exec.Runtimes
import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.utils.SparkSchemaUtil
@@ -38,13 +38,12 @@ object DatasourceUtil {
def readSchema(file: FileStatus): Option[StructType] = {
val allocator = ArrowBufferAllocators.contextInstance()
- val datasourceJniWrapper = DatasourceJniWrapper.create()
+ val runtime = Runtimes.contextInstance("VeloxWriter")
+ val datasourceJniWrapper = DatasourceJniWrapper.create(runtime)
val dsHandle = datasourceJniWrapper.nativeInitDatasource(
file.getPath.toString,
-1,
- NativeMemoryManagers.contextInstance("VeloxWriter").getNativeInstanceHandle,
- new util.HashMap[String, String]()
- )
+ new util.HashMap[String, String]())
val cSchema = ArrowSchema.allocateNew(allocator)
datasourceJniWrapper.inspectSchema(dsHandle, cSchema.memoryAddress())
try {
diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala
index 6346251b6..3d532133a 100644
--- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala
+++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/BroadcastUtils.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.execution
import org.apache.gluten.columnarbatch.ColumnarBatches
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
+import org.apache.gluten.exec.Runtimes
import org.apache.gluten.sql.shims.SparkShimLoader
import org.apache.gluten.vectorized.{ColumnarBatchSerializeResult, ColumnarBatchSerializerJniWrapper}
@@ -31,7 +31,6 @@ import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.vectorized.ColumnarBatch
import org.apache.spark.util.TaskResources
-import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer;
// Utility methods to convert Vanilla broadcast relations from/to Velox broadcast relations.
@@ -153,17 +152,12 @@ object BroadcastUtils {
if (filtered.isEmpty) {
return ColumnarBatchSerializeResult.EMPTY
}
- val batchRuntime = ColumnarBatches.getRuntime(filtered.toList.asJava)
val handleArray = filtered.map(ColumnarBatches.getNativeHandle)
val serializeResult =
try {
ColumnarBatchSerializerJniWrapper
- .forRuntime(batchRuntime)
- .serialize(
- handleArray,
- NativeMemoryManagers
- .contextInstance("BroadcastRelation")
- .getNativeInstanceHandle)
+ .create(Runtimes.contextInstance("BroadcastUtils#serializeStream"))
+ .serialize(handleArray)
} finally {
filtered.foreach(ColumnarBatches.release)
}
diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
index cb65b7504..3f82f919b 100644
--- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
+++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/ColumnarCachedBatchSerializer.scala
@@ -22,7 +22,6 @@ import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.exec.Runtimes
import org.apache.gluten.execution.{RowToVeloxColumnarExec, VeloxColumnarToRowExec}
import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.ArrowAbiUtil
import org.apache.gluten.utils.iterator.Iterators
import org.apache.gluten.vectorized.ColumnarBatchSerializerJniWrapper
@@ -150,8 +149,7 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe
numInputRows,
numOutputBatches,
convertTime,
- numRows
- )
+ numRows)
}
convertColumnarBatchToCachedBatch(rddColumnarBatch, schema, storageLevel, conf)
}
@@ -186,8 +184,7 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe
selectedAttributes,
numOutputRows,
numInputBatches,
- convertTime
- )
+ convertTime)
}
}
@@ -198,10 +195,6 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe
conf: SQLConf): RDD[CachedBatch] = {
input.mapPartitions {
it =>
- val nativeMemoryManagerHandle = NativeMemoryManagers
- .contextInstance("ColumnarCachedBatchSerializer serialize")
- .getNativeInstanceHandle
-
new Iterator[CachedBatch] {
override def hasNext: Boolean = it.hasNext
@@ -209,11 +202,8 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe
val batch = it.next()
val results =
ColumnarBatchSerializerJniWrapper
- .create()
- .serialize(
- Array(ColumnarBatches.getNativeHandle(batch)),
- nativeMemoryManagerHandle
- )
+ .create(Runtimes.contextInstance("ColumnarCachedBatchSerializer#serialize"))
+ .serialize(Array(ColumnarBatches.getNativeHandle(batch)))
CachedColumnarBatch(
results.getNumRows.toInt,
results.getSerialized.length,
@@ -237,19 +227,15 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe
val timezoneId = SQLConf.get.sessionLocalTimeZone
input.mapPartitions {
it =>
+ val runtime = Runtimes.contextInstance("ColumnarCachedBatchSerializer#read")
val jniWrapper = ColumnarBatchSerializerJniWrapper
- .create()
- val nmm = NativeMemoryManagers
- .contextInstance("ColumnarCachedBatchSerializer read")
+ .create(runtime)
val schema = SparkArrowUtil.toArrowSchema(localSchema, timezoneId)
val arrowAlloc = ArrowBufferAllocators.contextInstance()
val cSchema = ArrowSchema.allocateNew(arrowAlloc)
ArrowAbiUtil.exportSchema(arrowAlloc, schema, cSchema)
val deserializerHandle = jniWrapper
- .init(
- cSchema.memoryAddress(),
- nmm.getNativeInstanceHandle
- )
+ .init(cSchema.memoryAddress())
cSchema.close()
Iterators
@@ -261,10 +247,10 @@ class ColumnarCachedBatchSerializer extends CachedBatchSerializer with SQLConfHe
val batchHandle =
jniWrapper
.deserialize(deserializerHandle, cachedBatch.bytes)
- val batch = ColumnarBatches.create(Runtimes.contextInstance(), batchHandle)
+ val batch = ColumnarBatches.create(runtime, batchHandle)
if (shouldSelectAttributes) {
try {
- ColumnarBatches.select(nmm, batch, requestedColumnIndices.toArray)
+ ColumnarBatches.select(batch, requestedColumnIndices.toArray)
} finally {
batch.close()
}
diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java
index 4d887e607..56df7b9ad 100644
--- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java
+++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxBlockStripes.java
@@ -53,7 +53,8 @@ public class VeloxBlockStripes extends BlockStripes {
return new BlockStripe() {
@Override
public ColumnarBatch getColumnarBatch() {
- return ColumnarBatches.create(Runtimes.contextInstance(), blockAddresses[0]);
+ return ColumnarBatches.create(
+ Runtimes.contextInstance("VeloxBlockStripes"), blockAddresses[0]);
}
@Override
diff --git a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
index 9f9d43326..ebbf959d0 100644
--- a/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
+++ b/backends-velox/src/main/scala/org/apache/spark/sql/execution/datasources/velox/VeloxFormatWriterInjects.scala
@@ -19,9 +19,9 @@ package org.apache.spark.sql.execution.datasources.velox
import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.datasource.DatasourceJniWrapper
import org.apache.gluten.exception.GlutenException
+import org.apache.gluten.exec.Runtimes
import org.apache.gluten.execution.datasource.GlutenRowSplitter
import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.{ArrowAbiUtil, DatasourceUtil}
import org.apache.spark.sql.SparkSession
@@ -59,15 +59,13 @@ trait VeloxFormatWriterInjects extends GlutenFormatWriterInjectsBase {
SparkArrowUtil.toArrowSchema(dataSchema, SQLConf.get.sessionLocalTimeZone)
val cSchema = ArrowSchema.allocateNew(ArrowBufferAllocators.contextInstance())
var dsHandle = -1L
- val datasourceJniWrapper = DatasourceJniWrapper.create()
+ val runtime = Runtimes.contextInstance("VeloxWriter")
+ val datasourceJniWrapper = DatasourceJniWrapper.create(runtime)
val allocator = ArrowBufferAllocators.contextInstance()
try {
ArrowAbiUtil.exportSchema(allocator, arrowSchema, cSchema)
- dsHandle = datasourceJniWrapper.nativeInitDatasource(
- filePath,
- cSchema.memoryAddress(),
- NativeMemoryManagers.contextInstance("VeloxWriter").getNativeInstanceHandle,
- nativeConf)
+ dsHandle =
+ datasourceJniWrapper.nativeInitDatasource(filePath, cSchema.memoryAddress(), nativeConf)
} catch {
case e: IOException =>
throw new GlutenException(e)
@@ -119,16 +117,12 @@ class VeloxRowSplitter extends GlutenRowSplitter {
hasBucket: Boolean,
reserve_partition_columns: Boolean = false): BlockStripes = {
val handler = ColumnarBatches.getNativeHandle(row.batch)
- val datasourceJniWrapper = DatasourceJniWrapper.create()
+ val runtime = Runtimes.contextInstance("VeloxPartitionWriter")
+ val datasourceJniWrapper = DatasourceJniWrapper.create(runtime)
val originalColumns: Array[Int] = Array.range(0, row.batch.numCols())
val dataColIndice = originalColumns.filterNot(partitionColIndice.contains(_))
new VeloxBlockStripes(
datasourceJniWrapper
- .splitBlockByPartitionAndBucket(
- handler,
- dataColIndice,
- hasBucket,
- NativeMemoryManagers.contextInstance("VeloxPartitionWriter").getNativeInstanceHandle)
- )
+ .splitBlockByPartitionAndBucket(handler, dataColIndice, hasBucket))
}
}
diff --git a/cpp/core/compute/Runtime.cc b/cpp/core/compute/Runtime.cc
index 387da1a0a..c6bae1e7b 100644
--- a/cpp/core/compute/Runtime.cc
+++ b/cpp/core/compute/Runtime.cc
@@ -56,9 +56,12 @@ void Runtime::registerFactory(const std::string& kind, Runtime::Factory factory)
runtimeFactories().registerFactory(kind, std::move(factory));
}
-Runtime* Runtime::create(const std::string& kind, const std::unordered_map<std::string, std::string>& sessionConf) {
+Runtime* Runtime::create(
+ const std::string& kind,
+ std::unique_ptr<AllocationListener> listener,
+ const std::unordered_map<std::string, std::string>& sessionConf) {
auto& factory = runtimeFactories().getFactory(kind);
- return factory(sessionConf);
+ return factory(std::move(listener), sessionConf);
}
void Runtime::release(Runtime* runtime) {
diff --git a/cpp/core/compute/Runtime.h b/cpp/core/compute/Runtime.h
index 7574b0219..fb501dc9a 100644
--- a/cpp/core/compute/Runtime.h
+++ b/cpp/core/compute/Runtime.h
@@ -55,13 +55,18 @@ struct SparkTaskInfo {
class Runtime : public std::enable_shared_from_this<Runtime> {
public:
- using Factory = std::function<Runtime*(const std::unordered_map<std::string, std::string>&)>;
+ using Factory = std::function<
+ Runtime*(std::unique_ptr<AllocationListener> listener, const std::unordered_map<std::string, std::string>&)>;
static void registerFactory(const std::string& kind, Factory factory);
- static Runtime* create(const std::string& kind, const std::unordered_map<std::string, std::string>& sessionConf = {});
+ static Runtime* create(
+ const std::string& kind,
+ std::unique_ptr<AllocationListener> listener,
+ const std::unordered_map<std::string, std::string>& sessionConf = {});
static void release(Runtime*);
- Runtime() = default;
- Runtime(const std::unordered_map<std::string, std::string>& confMap) : confMap_(confMap) {}
+ Runtime(std::shared_ptr<MemoryManager> memoryManager, const std::unordered_map<std::string, std::string>& confMap)
+ : memoryManager_(memoryManager), confMap_(confMap) {}
+
virtual ~Runtime() = default;
virtual void parsePlan(const uint8_t* data, int32_t size, std::optional<std::string> dumpFile) = 0;
@@ -78,52 +83,40 @@ class Runtime : public std::enable_shared_from_this<Runtime> {
}
virtual std::shared_ptr<ResultIterator> createResultIterator(
- MemoryManager* memoryManager,
const std::string& spillDir,
const std::vector<std::shared_ptr<ResultIterator>>& inputs,
const std::unordered_map<std::string, std::string>& sessionConf) = 0;
virtual std::shared_ptr<ColumnarBatch> createOrGetEmptySchemaBatch(int32_t numRows) = 0;
- virtual std::shared_ptr<ColumnarBatch>
- select(MemoryManager*, std::shared_ptr<ColumnarBatch>, std::vector<int32_t>) = 0;
+ virtual std::shared_ptr<ColumnarBatch> select(std::shared_ptr<ColumnarBatch>, std::vector<int32_t>) = 0;
- virtual MemoryManager* createMemoryManager(
- const std::string& name,
- std::shared_ptr<MemoryAllocator>,
- std::unique_ptr<AllocationListener>) = 0;
+ virtual MemoryManager* memoryManager() {
+ return memoryManager_.get();
+ };
/// This function is used to create certain converter from the format used by
/// the backend to Spark unsafe row.
- virtual std::shared_ptr<ColumnarToRowConverter> createColumnar2RowConverter(MemoryManager* memoryManager) = 0;
+ virtual std::shared_ptr<ColumnarToRowConverter> createColumnar2RowConverter() = 0;
- virtual std::shared_ptr<RowToColumnarConverter> createRow2ColumnarConverter(
- MemoryManager* memoryManager,
- struct ArrowSchema* cSchema) = 0;
+ virtual std::shared_ptr<RowToColumnarConverter> createRow2ColumnarConverter(struct ArrowSchema* cSchema) = 0;
virtual std::shared_ptr<ShuffleWriter> createShuffleWriter(
int numPartitions,
std::unique_ptr<PartitionWriter> partitionWriter,
- ShuffleWriterOptions options,
- MemoryManager* memoryManager) = 0;
+ ShuffleWriterOptions options) = 0;
virtual Metrics* getMetrics(ColumnarBatchIterator* rawIter, int64_t exportNanos) = 0;
virtual std::shared_ptr<Datasource> createDatasource(
const std::string& filePath,
- MemoryManager* memoryManager,
std::shared_ptr<arrow::Schema> schema) = 0;
virtual std::shared_ptr<ShuffleReader> createShuffleReader(
std::shared_ptr<arrow::Schema> schema,
- ShuffleReaderOptions options,
- arrow::MemoryPool* pool,
- MemoryManager* memoryManager) = 0;
+ ShuffleReaderOptions options) = 0;
- virtual std::unique_ptr<ColumnarBatchSerializer> createColumnarBatchSerializer(
- MemoryManager* memoryManager,
- arrow::MemoryPool* arrowPool,
- struct ArrowSchema* cSchema) = 0;
+ virtual std::unique_ptr<ColumnarBatchSerializer> createColumnarBatchSerializer(struct ArrowSchema* cSchema) = 0;
virtual void dumpConf(const std::string& path) = 0;
@@ -135,17 +128,18 @@ class Runtime : public std::enable_shared_from_this<Runtime> {
taskInfo_ = taskInfo;
}
- ObjectStore* objectStore() {
- return objStore_.get();
+ ObjectHandle saveObject(std::shared_ptr<void> obj) {
+ return objStore_->save(obj);
}
protected:
+ std::shared_ptr<MemoryManager> memoryManager_;
std::unique_ptr<ObjectStore> objStore_ = ObjectStore::create();
+ std::unordered_map<std::string, std::string> confMap_; // Session conf map
+
::substrait::Plan substraitPlan_;
std::vector<::substrait::ReadRel_LocalFiles> localFiles_;
std::optional<std::string> writeFilesTempPath_;
SparkTaskInfo taskInfo_;
- // Session conf map
- std::unordered_map<std::string, std::string> confMap_;
};
} // namespace gluten
diff --git a/cpp/core/jni/JniCommon.cc b/cpp/core/jni/JniCommon.cc
index 08c5cb1d4..759a9d121 100644
--- a/cpp/core/jni/JniCommon.cc
+++ b/cpp/core/jni/JniCommon.cc
@@ -112,7 +112,7 @@ std::shared_ptr<gluten::ColumnarBatch> gluten::JniColumnarBatchIterator::next()
checkException(env);
jlong handle = env->CallLongMethod(jColumnarBatchItr_, serializedColumnarBatchIteratorNext_);
checkException(env);
- auto batch = runtime_->objectStore()->retrieve<ColumnarBatch>(handle);
+ auto batch = ObjectStore::retrieve<ColumnarBatch>(handle);
if (writer_ != nullptr) {
// save snapshot of the batch to file
std::shared_ptr<ArrowSchema> schema = batch->exportArrowSchema();
diff --git a/cpp/core/jni/JniCommon.h b/cpp/core/jni/JniCommon.h
index bc5cf84f6..d5c9f2b3b 100644
--- a/cpp/core/jni/JniCommon.h
+++ b/cpp/core/jni/JniCommon.h
@@ -28,6 +28,7 @@
#include "memory/AllocationListener.h"
#include "shuffle/rss/RssClient.h"
#include "utils/Compression.h"
+#include "utils/ObjectStore.h"
#include "utils/ResourceMap.h"
#include "utils/exception.h"
@@ -121,7 +122,7 @@ static inline void attachCurrentThreadAsDaemonOrThrow(JavaVM* vm, JNIEnv** out)
}
template <typename T>
-static T* jniCastOrThrow(gluten::ResourceHandle handle) {
+static T* jniCastOrThrow(jlong handle) {
auto instance = reinterpret_cast<T*>(handle);
GLUTEN_CHECK(instance != nullptr, "FATAL: resource instance should not be null.");
return instance;
diff --git a/cpp/core/jni/JniError.cc b/cpp/core/jni/JniError.cc
index af21ba2a1..b0f4e713a 100644
--- a/cpp/core/jni/JniError.cc
+++ b/cpp/core/jni/JniError.cc
@@ -41,7 +41,13 @@ jclass gluten::JniErrorState::illegalAccessExceptionClass() {
return illegalAccessExceptionClass_;
}
+jclass gluten::JniErrorState::glutenExceptionClass() {
+ assertInitialized();
+ return glutenExceptionClass_;
+}
+
void gluten::JniErrorState::initialize(JNIEnv* env) {
+ glutenExceptionClass_ = createGlobalClassReference(env, "Lorg/apache/gluten/exception/GlutenException;");
ioExceptionClass_ = createGlobalClassReference(env, "Ljava/io/IOException;");
runtimeExceptionClass_ = createGlobalClassReference(env, "Ljava/lang/RuntimeException;");
unsupportedOperationExceptionClass_ = createGlobalClassReference(env, "Ljava/lang/UnsupportedOperationException;");
@@ -61,6 +67,7 @@ void gluten::JniErrorState::close() {
}
JNIEnv* env;
attachCurrentThreadAsDaemonOrThrow(vm_, &env);
+ env->DeleteGlobalRef(glutenExceptionClass_);
env->DeleteGlobalRef(ioExceptionClass_);
env->DeleteGlobalRef(runtimeExceptionClass_);
env->DeleteGlobalRef(unsupportedOperationExceptionClass_);
diff --git a/cpp/core/jni/JniError.h b/cpp/core/jni/JniError.h
index f902ebe03..828c90861 100644
--- a/cpp/core/jni/JniError.h
+++ b/cpp/core/jni/JniError.h
@@ -28,11 +28,11 @@
#endif
#ifndef JNI_METHOD_END
-#define JNI_METHOD_END(fallback_expr) \
- } \
- catch (std::exception & e) { \
- env->ThrowNew(gluten::getJniErrorState()->runtimeExceptionClass(), e.what()); \
- return fallback_expr; \
+#define JNI_METHOD_END(fallback_expr) \
+ } \
+ catch (std::exception & e) { \
+ env->ThrowNew(gluten::getJniErrorState()->glutenExceptionClass(), e.what()); \
+ return fallback_expr; \
}
// macro ended
#endif
@@ -53,6 +53,8 @@ class JniErrorState {
jclass illegalAccessExceptionClass();
+ jclass glutenExceptionClass();
+
private:
void initialize(JNIEnv* env);
@@ -61,6 +63,7 @@ class JniErrorState {
jclass unsupportedOperationExceptionClass_ = nullptr;
jclass illegalAccessExceptionClass_ = nullptr;
jclass illegalArgumentExceptionClass_ = nullptr;
+ jclass glutenExceptionClass_ = nullptr;
JavaVM* vm_;
bool initialized_{false};
bool closed_{false};
diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc
index 1e5326689..23eea2db7 100644
--- a/cpp/core/jni/JniWrapper.cc
+++ b/cpp/core/jni/JniWrapper.cc
@@ -180,7 +180,7 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) {
javaReservationListenerClass = createGlobalClassReference(
env,
- "Lorg/apache/gluten/memory/nmm/"
+ "Lorg/apache/gluten/memory/listener/"
"ReservationListener;");
reserveMemoryMethod = getMethodIdOrError(env, javaReservationListenerClass, "reserve", "(J)J");
@@ -219,14 +219,63 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_createRunt
JNIEnv* env,
jclass,
jstring jbackendType,
+ jobject jlistener,
jbyteArray sessionConf) {
JNI_METHOD_START
+ JavaVM* vm;
+ if (env->GetJavaVM(&vm) != JNI_OK) {
+ throw gluten::GlutenException("Unable to get JavaVM instance");
+ }
+
auto backendType = jStringToCString(env, jbackendType);
+ std::unique_ptr<AllocationListener> listener =
+ std::make_unique<SparkAllocationListener>(vm, jlistener, reserveMemoryMethod, unreserveMemoryMethod);
+
auto safeArray = gluten::getByteArrayElementsSafe(env, sessionConf);
auto sparkConf = gluten::parseConfMap(env, safeArray.elems(), safeArray.length());
- auto runtime = gluten::Runtime::create(backendType, sparkConf);
+ auto runtime = gluten::Runtime::create(backendType, std::move(listener), sparkConf);
return reinterpret_cast<jlong>(runtime);
- JNI_METHOD_END(kInvalidResourceHandle)
+ JNI_METHOD_END(kInvalidObjectHandle)
+}
+
+JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_collectMemoryUsage( // NOLINT
+ JNIEnv* env,
+ jclass,
+ jlong ctxHandle) {
+ JNI_METHOD_START
+ auto runtime = jniCastOrThrow<Runtime>(ctxHandle);
+
+ const MemoryUsageStats& stats = runtime->memoryManager()->collectMemoryUsageStats();
+ auto size = stats.ByteSizeLong();
+ jbyteArray out = env->NewByteArray(size);
+ uint8_t buffer[size];
+ GLUTEN_CHECK(
+ stats.SerializeToArray(reinterpret_cast<void*>(buffer), size),
+ "Serialization failed when collecting memory usage stats");
+ env->SetByteArrayRegion(out, 0, size, reinterpret_cast<jbyte*>(buffer));
+ return out;
+ JNI_METHOD_END(nullptr)
+}
+
+JNIEXPORT jlong JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_shrinkMemory( // NOLINT
+ JNIEnv* env,
+ jclass,
+ jlong ctxHandle,
+ jlong size) {
+ JNI_METHOD_START
+ auto runtime = jniCastOrThrow<Runtime>(ctxHandle);
+ return runtime->memoryManager()->shrink(static_cast<int64_t>(size));
+ JNI_METHOD_END(kInvalidObjectHandle)
+}
+
+JNIEXPORT void JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_holdMemory( // NOLINT
+ JNIEnv* env,
+ jclass,
+ jlong ctxHandle) {
+ JNI_METHOD_START
+ auto runtime = jniCastOrThrow<Runtime>(ctxHandle);
+ runtime->memoryManager()->hold();
+ JNI_METHOD_END()
}
JNIEXPORT void JNICALL Java_org_apache_gluten_exec_RuntimeJniWrapper_releaseRuntime( // NOLINT
@@ -278,7 +327,6 @@ JNIEXPORT jlong JNICALL
Java_org_apache_gluten_vectorized_PlanEvaluatorJniWrapper_nativeCreateKernelWithIterator( // NOLINT
JNIEnv* env,
jobject wrapper,
- jlong memoryManagerHandle,
jbyteArray planArr,
jobjectArray splitInfosArr,
jobjectArray iterArr,
@@ -308,8 +356,6 @@ Java_org_apache_gluten_vectorized_PlanEvaluatorJniWrapper_nativeCreateKernelWith
ctx->dumpConf(saveDir + "/conf" + fileIdentifier + ".ini");
}
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
-
auto spillDirStr = jStringToCString(env, spillDir);
auto safePlanArray = gluten::getByteArrayElementsSafe(env, planArr);
@@ -346,8 +392,8 @@ Java_org_apache_gluten_vectorized_PlanEvaluatorJniWrapper_nativeCreateKernelWith
inputIters.push_back(std::move(resultIter));
}
- return ctx->objectStore()->save(ctx->createResultIterator(memoryManager, spillDirStr, inputIters, conf));
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(ctx->createResultIterator(spillDirStr, inputIters, conf));
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jboolean JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterator_nativeHasNext( // NOLINT
@@ -357,7 +403,7 @@ JNIEXPORT jboolean JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIte
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto iter = ctx->objectStore()->retrieve<ResultIterator>(iterHandle);
+ auto iter = ObjectStore::retrieve<ResultIterator>(iterHandle);
if (iter == nullptr) {
std::string errorMessage =
"When hasNext() is called on a closed iterator, an exception is thrown. To prevent this, consider using the protectInvocationFlow() method when creating the iterator in scala side. This will allow the hasNext() method to be called multiple times without issue.";
@@ -374,17 +420,17 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterat
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto iter = ctx->objectStore()->retrieve<ResultIterator>(iterHandle);
+ auto iter = ObjectStore::retrieve<ResultIterator>(iterHandle);
if (!iter->hasNext()) {
- return kInvalidResourceHandle;
+ return kInvalidObjectHandle;
}
std::shared_ptr<ColumnarBatch> batch = iter->next();
- auto batchHandle = ctx->objectStore()->save(batch);
+ auto batchHandle = ctx->saveObject(batch);
iter->setExportNanos(batch->getExportNanos());
return batchHandle;
- JNI_METHOD_END(kInvalidResourceHandle)
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterator_nativeFetchMetrics( // NOLINT
@@ -394,7 +440,7 @@ JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIter
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto iter = ctx->objectStore()->retrieve<ResultIterator>(iterHandle);
+ auto iter = ObjectStore::retrieve<ResultIterator>(iterHandle);
auto metrics = iter->getMetrics();
unsigned int numMetrics = 0;
if (metrics) {
@@ -455,13 +501,13 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterat
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto it = ctx->objectStore()->retrieve<ResultIterator>(iterHandle);
+ auto it = ObjectStore::retrieve<ResultIterator>(iterHandle);
if (it == nullptr) {
std::string errorMessage = "Invalid result iter handle " + std::to_string(iterHandle);
throw gluten::GlutenException(errorMessage);
}
return it->spillFixedSize(size);
- JNI_METHOD_END(kInvalidResourceHandle)
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterator_nativeClose( // NOLINT
@@ -471,34 +517,32 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchOutIterato
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- ctx->objectStore()->release(iterHandle);
+ ObjectStore::release(iterHandle);
JNI_METHOD_END()
}
JNIEXPORT jlong JNICALL
Java_org_apache_gluten_vectorized_NativeColumnarToRowJniWrapper_nativeColumnarToRowInit( // NOLINT
JNIEnv* env,
- jobject wrapper,
- jlong memoryManagerHandle) {
+ jobject wrapper) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
// Convert the native batch to Spark unsafe row.
- return ctx->objectStore()->save(ctx->createColumnar2RowConverter(memoryManager));
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(ctx->createColumnar2RowConverter());
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jobject JNICALL
Java_org_apache_gluten_vectorized_NativeColumnarToRowJniWrapper_nativeColumnarToRowConvert( // NOLINT
JNIEnv* env,
jobject wrapper,
- jlong batchHandle,
- jlong c2rHandle) {
+ jlong c2rHandle,
+ jlong batchHandle) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto columnarToRowConverter = ctx->objectStore()->retrieve<ColumnarToRowConverter>(c2rHandle);
- auto cb = ctx->objectStore()->retrieve<ColumnarBatch>(batchHandle);
+ auto columnarToRowConverter = ObjectStore::retrieve<ColumnarToRowConverter>(c2rHandle);
+ auto cb = ObjectStore::retrieve<ColumnarBatch>(batchHandle);
columnarToRowConverter->convert(cb);
const auto& offsets = columnarToRowConverter->getOffsets();
@@ -527,22 +571,19 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_NativeColumnarToRowJniW
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- ctx->objectStore()->release(c2rHandle);
+ ObjectStore::release(c2rHandle);
JNI_METHOD_END()
}
JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_NativeRowToColumnarJniWrapper_init( // NOLINT
JNIEnv* env,
jobject wrapper,
- jlong cSchema,
- jlong memoryManagerHandle) {
+ jlong cSchema) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
- return ctx->objectStore()->save(
- ctx->createRow2ColumnarConverter(memoryManager, reinterpret_cast<struct ArrowSchema*>(cSchema)));
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(ctx->createRow2ColumnarConverter(reinterpret_cast<struct ArrowSchema*>(cSchema)));
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jlong JNICALL
@@ -562,10 +603,10 @@ Java_org_apache_gluten_vectorized_NativeRowToColumnarJniWrapper_nativeConvertRow
auto safeArray = gluten::getLongArrayElementsSafe(env, rowLength);
uint8_t* address = reinterpret_cast<uint8_t*>(memoryAddress);
- auto converter = ctx->objectStore()->retrieve<RowToColumnarConverter>(r2cHandle);
+ auto converter = ObjectStore::retrieve<RowToColumnarConverter>(r2cHandle);
auto cb = converter->convert(numRows, safeArray.elems(), address);
- return ctx->objectStore()->save(cb);
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(cb);
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_NativeRowToColumnarJniWrapper_close( // NOLINT
@@ -575,7 +616,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_NativeRowToColumnarJniW
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- ctx->objectStore()->release(r2cHandle);
+ ObjectStore::release(r2cHandle);
JNI_METHOD_END()
}
@@ -585,7 +626,7 @@ JNIEXPORT jstring JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniW
jlong batchHandle) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto batch = ctx->objectStore()->retrieve<ColumnarBatch>(batchHandle);
+ auto batch = ObjectStore::retrieve<ColumnarBatch>(batchHandle);
return env->NewStringUTF(batch->getType().c_str());
JNI_METHOD_END(nullptr)
}
@@ -596,9 +637,9 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra
jlong batchHandle) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto batch = ctx->objectStore()->retrieve<ColumnarBatch>(batchHandle);
+ auto batch = ObjectStore::retrieve<ColumnarBatch>(batchHandle);
return batch->numBytes();
- JNI_METHOD_END(kInvalidResourceHandle)
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_numColumns( // NOLINT
@@ -607,9 +648,9 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra
jlong batchHandle) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto batch = ctx->objectStore()->retrieve<ColumnarBatch>(batchHandle);
+ auto batch = ObjectStore::retrieve<ColumnarBatch>(batchHandle);
return batch->numColumns();
- JNI_METHOD_END(kInvalidResourceHandle)
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_numRows( // NOLINT
@@ -618,9 +659,9 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra
jlong batchHandle) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto batch = ctx->objectStore()->retrieve<ColumnarBatch>(batchHandle);
+ auto batch = ObjectStore::retrieve<ColumnarBatch>(batchHandle);
return batch->numRows();
- JNI_METHOD_END(kInvalidResourceHandle)
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_compose( // NOLINT
@@ -636,12 +677,12 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra
std::vector<std::shared_ptr<ColumnarBatch>> batches;
for (int i = 0; i < handleCount; ++i) {
int64_t handle = safeArray.elems()[i];
- auto batch = ctx->objectStore()->retrieve<ColumnarBatch>(handle);
+ auto batch = ObjectStore::retrieve<ColumnarBatch>(handle);
batches.push_back(batch);
}
auto newBatch = CompositeColumnarBatch::create(std::move(batches));
- return ctx->objectStore()->save(newBatch);
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(newBatch);
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT void JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_exportToArrow( // NOLINT
@@ -652,7 +693,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrap
jlong cArray) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto batch = ctx->objectStore()->retrieve<ColumnarBatch>(batchHandle);
+ auto batch = ObjectStore::retrieve<ColumnarBatch>(batchHandle);
std::shared_ptr<ArrowSchema> exportedSchema = batch->exportArrowSchema();
std::shared_ptr<ArrowArray> exportedArray = batch->exportArrowArray();
ArrowSchemaMove(exportedSchema.get(), reinterpret_cast<struct ArrowSchema*>(cSchema));
@@ -676,8 +717,8 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra
ArrowSchemaMove(arrowSchema, targetSchema.get());
std::shared_ptr<ColumnarBatch> batch =
std::make_shared<ArrowCStructColumnarBatch>(std::move(targetSchema), std::move(targetArray));
- return ctx->objectStore()->save(batch);
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(batch);
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_getForEmptySchema( // NOLINT
@@ -686,19 +727,17 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra
jint numRows) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- return ctx->objectStore()->save(ctx->createOrGetEmptySchemaBatch(static_cast<int32_t>(numRows)));
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(ctx->createOrGetEmptySchemaBatch(static_cast<int32_t>(numRows)));
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_select( // NOLINT
JNIEnv* env,
jobject wrapper,
- jlong memoryManagerHandle,
jlong batchHandle,
jintArray jcolumnIndices) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
auto safeArray = gluten::getIntArrayElementsSafe(env, jcolumnIndices);
int size = env->GetArrayLength(jcolumnIndices);
@@ -707,9 +746,20 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWra
columnIndices.push_back(safeArray.elems()[i]);
}
- return ctx->objectStore()->save(
- ctx->select(memoryManager, ctx->objectStore()->retrieve<ColumnarBatch>(batchHandle), std::move(columnIndices)));
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(ctx->select(ObjectStore::retrieve<ColumnarBatch>(batchHandle), std::move(columnIndices)));
+ JNI_METHOD_END(kInvalidObjectHandle)
+}
+
+JNIEXPORT jlong JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_obtainOwnership( // NOLINT
+ JNIEnv* env,
+ jobject wrapper,
+ jlong batchHandle) {
+ JNI_METHOD_START
+ auto ctx = gluten::getRuntime(env, wrapper);
+ auto batch = ObjectStore::retrieve<ColumnarBatch>(batchHandle);
+ auto newHandle = ctx->saveObject(batch);
+ return newHandle;
+ JNI_METHOD_END(-1L)
}
JNIEXPORT void JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrapper_close( // NOLINT
@@ -718,7 +768,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_columnarbatch_ColumnarBatchJniWrap
jlong batchHandle) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- ctx->objectStore()->release(batchHandle);
+ ObjectStore::release(batchHandle);
JNI_METHOD_END()
}
@@ -739,7 +789,6 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe
jstring dataFileJstr,
jint numSubDirs,
jstring localDirsJstr,
- jlong memoryManagerHandle,
jdouble reallocThreshold,
jlong firstBatchHandle,
jlong taskAttemptId,
@@ -751,7 +800,6 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe
jstring shuffleWriterTypeJstr) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
if (partitioningNameJstr == nullptr) {
throw gluten::GlutenException(std::string("Short partitioning name can't be null"));
}
@@ -813,7 +861,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe
partitionWriter = std::make_unique<LocalPartitionWriter>(
numPartitions,
std::move(partitionWriterOptions),
- memoryManager->getArrowMemoryPool(),
+ ctx->memoryManager()->getArrowMemoryPool(),
dataFile,
configuredDirs);
} else if (partitionWriterType == "celeborn") {
@@ -830,7 +878,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe
partitionWriter = std::make_unique<RssPartitionWriter>(
numPartitions,
std::move(partitionWriterOptions),
- memoryManager->getArrowMemoryPool(),
+ ctx->memoryManager()->getArrowMemoryPool(),
std::move(celebornClient));
} else if (partitionWriterType == "uniffle") {
jclass unifflePartitionPusherClass =
@@ -846,15 +894,15 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe
partitionWriter = std::make_unique<RssPartitionWriter>(
numPartitions,
std::move(partitionWriterOptions),
- memoryManager->getArrowMemoryPool(),
+ ctx->memoryManager()->getArrowMemoryPool(),
std::move(uniffleClient));
} else {
throw gluten::GlutenException("Unrecognizable partition writer type: " + partitionWriterType);
}
- return ctx->objectStore()->save(ctx->createShuffleWriter(
- numPartitions, std::move(partitionWriter), std::move(shuffleWriterOptions), memoryManager));
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(
+ ctx->createShuffleWriter(numPartitions, std::move(partitionWriter), std::move(shuffleWriterOptions)));
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrapper_nativeEvict( // NOLINT
@@ -866,7 +914,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto shuffleWriter = ctx->objectStore()->retrieve<ShuffleWriter>(shuffleWriterHandle);
+ auto shuffleWriter = ObjectStore::retrieve<ShuffleWriter>(shuffleWriterHandle);
if (!shuffleWriter) {
std::string errorMessage = "Invalid shuffle writer handle " + std::to_string(shuffleWriterHandle);
throw gluten::GlutenException(errorMessage);
@@ -875,7 +923,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe
gluten::arrowAssertOkOrThrow(
shuffleWriter->reclaimFixedSize(size, &evictedSize), "(shuffle) nativeEvict: evict failed");
return (jlong)evictedSize;
- JNI_METHOD_END(kInvalidResourceHandle)
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrapper_write( // NOLINT
@@ -888,18 +936,18 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrappe
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto shuffleWriter = ctx->objectStore()->retrieve<ShuffleWriter>(shuffleWriterHandle);
+ auto shuffleWriter = ObjectStore::retrieve<ShuffleWriter>(shuffleWriterHandle);
if (!shuffleWriter) {
std::string errorMessage = "Invalid shuffle writer handle " + std::to_string(shuffleWriterHandle);
throw gluten::GlutenException(errorMessage);
}
// The column batch maybe VeloxColumnBatch or ArrowCStructColumnarBatch(FallbackRangeShuffleWriter)
- auto batch = ctx->objectStore()->retrieve<ColumnarBatch>(batchHandle);
+ auto batch = ObjectStore::retrieve<ColumnarBatch>(batchHandle);
auto numBytes = batch->numBytes();
gluten::arrowAssertOkOrThrow(shuffleWriter->write(batch, memLimit), "Native write: shuffle writer failed");
return numBytes;
- JNI_METHOD_END(kInvalidResourceHandle)
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrapper_stop( // NOLINT
@@ -909,7 +957,7 @@ JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrap
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto shuffleWriter = ctx->objectStore()->retrieve<ShuffleWriter>(shuffleWriterHandle);
+ auto shuffleWriter = ObjectStore::retrieve<ShuffleWriter>(shuffleWriterHandle);
if (!shuffleWriter) {
std::string errorMessage = "Invalid shuffle writer handle " + std::to_string(shuffleWriterHandle);
throw gluten::GlutenException(errorMessage);
@@ -951,7 +999,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ShuffleWriterJniWrapper
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- ctx->objectStore()->release(shuffleWriterHandle);
+ ObjectStore::release(shuffleWriterHandle);
JNI_METHOD_END()
}
@@ -971,16 +1019,13 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrappe
JNIEnv* env,
jobject wrapper,
jlong cSchema,
- jlong memoryManagerHandle,
jstring compressionType,
jstring compressionBackend,
jint batchSize,
jstring shuffleWriterType) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
- auto pool = memoryManager->getArrowMemoryPool();
ShuffleReaderOptions options = ShuffleReaderOptions{};
options.compressionType = getCompressionType(env, compressionType);
options.compressionTypeStr = getCompressionTypeStr(env, compressionType);
@@ -996,8 +1041,8 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrappe
std::shared_ptr<arrow::Schema> schema =
gluten::arrowGetOrThrow(arrow::ImportSchema(reinterpret_cast<struct ArrowSchema*>(cSchema)));
- return ctx->objectStore()->save(ctx->createShuffleReader(schema, options, pool, memoryManager));
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(ctx->createShuffleReader(schema, options));
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrapper_readStream( // NOLINT
@@ -1007,11 +1052,11 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrappe
jobject jniIn) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto reader = ctx->objectStore()->retrieve<ShuffleReader>(shuffleReaderHandle);
+ auto reader = ObjectStore::retrieve<ShuffleReader>(shuffleReaderHandle);
std::shared_ptr<arrow::io::InputStream> in = std::make_shared<JavaInputStreamAdaptor>(env, reader->getPool(), jniIn);
auto outItr = reader->readStream(in);
- return ctx->objectStore()->save(outItr);
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(outItr);
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrapper_populateMetrics( // NOLINT
@@ -1022,7 +1067,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrapper
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto reader = ctx->objectStore()->retrieve<ShuffleReader>(shuffleReaderHandle);
+ auto reader = ObjectStore::retrieve<ShuffleReader>(shuffleReaderHandle);
env->CallVoidMethod(metrics, shuffleReaderMetricsSetDecompressTime, reader->getDecompressTime());
env->CallVoidMethod(metrics, shuffleReaderMetricsSetDeserializeTime, reader->getDeserializeTime());
@@ -1037,9 +1082,9 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ShuffleReaderJniWrapper
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto reader = ctx->objectStore()->retrieve<ShuffleReader>(shuffleReaderHandle);
+ auto reader = ObjectStore::retrieve<ShuffleReader>(shuffleReaderHandle);
GLUTEN_THROW_NOT_OK(reader->close());
- ctx->objectStore()->release(shuffleReaderHandle);
+ ObjectStore::release(shuffleReaderHandle);
JNI_METHOD_END()
}
@@ -1048,30 +1093,28 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_datasource_DatasourceJniWrapper_n
jobject wrapper,
jstring filePath,
jlong cSchema,
- jlong memoryManagerHandle,
jbyteArray options) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
- ResourceHandle handle = kInvalidResourceHandle;
+ ObjectHandle handle = kInvalidObjectHandle;
if (cSchema == -1) {
// Only inspect the schema and not write
- handle = ctx->objectStore()->save(ctx->createDatasource(jStringToCString(env, filePath), memoryManager, nullptr));
+ handle = ctx->saveObject(ctx->createDatasource(jStringToCString(env, filePath), nullptr));
} else {
auto safeArray = gluten::getByteArrayElementsSafe(env, options);
auto datasourceOptions = gluten::parseConfMap(env, safeArray.elems(), safeArray.length());
auto& sparkConf = ctx->getConfMap();
datasourceOptions.insert(sparkConf.begin(), sparkConf.end());
auto schema = gluten::arrowGetOrThrow(arrow::ImportSchema(reinterpret_cast<struct ArrowSchema*>(cSchema)));
- handle = ctx->objectStore()->save(ctx->createDatasource(jStringToCString(env, filePath), memoryManager, schema));
- auto datasource = ctx->objectStore()->retrieve<Datasource>(handle);
+ handle = ctx->saveObject(ctx->createDatasource(jStringToCString(env, filePath), schema));
+ auto datasource = ObjectStore::retrieve<Datasource>(handle);
datasource->init(datasourceOptions);
}
return handle;
- JNI_METHOD_END(kInvalidResourceHandle)
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT void JNICALL Java_org_apache_gluten_datasource_DatasourceJniWrapper_inspectSchema( // NOLINT
@@ -1082,7 +1125,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_datasource_DatasourceJniWrapper_in
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto datasource = ctx->objectStore()->retrieve<Datasource>(dsHandle);
+ auto datasource = ObjectStore::retrieve<Datasource>(dsHandle);
datasource->inspectSchema(reinterpret_cast<struct ArrowSchema*>(cSchema));
JNI_METHOD_END()
}
@@ -1094,9 +1137,9 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_datasource_DatasourceJniWrapper_cl
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto datasource = ctx->objectStore()->retrieve<Datasource>(dsHandle);
+ auto datasource = ObjectStore::retrieve<Datasource>(dsHandle);
datasource->close();
- ctx->objectStore()->release(dsHandle);
+ ObjectStore::release(dsHandle);
JNI_METHOD_END()
}
@@ -1107,7 +1150,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_datasource_DatasourceJniWrapper_wr
jobject jIter) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto datasource = ctx->objectStore()->retrieve<Datasource>(dsHandle);
+ auto datasource = ObjectStore::retrieve<Datasource>(dsHandle);
auto iter = makeJniColumnarBatchIterator(env, jIter, ctx, nullptr);
while (true) {
auto batch = iter->next();
@@ -1129,7 +1172,7 @@ Java_org_apache_gluten_datasource_DatasourceJniWrapper_splitBlockByPartitionAndB
jlong memoryManagerId) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto batch = ctx->objectStore()->retrieve<ColumnarBatch>(batchHandle);
+ auto batch = ObjectStore::retrieve<ColumnarBatch>(batchHandle);
auto safeArray = gluten::getIntArrayElementsSafe(env, partitionColIndice);
int size = env->GetArrayLength(partitionColIndice);
std::vector<int32_t> partitionColIndiceVec;
@@ -1137,10 +1180,9 @@ Java_org_apache_gluten_datasource_DatasourceJniWrapper_splitBlockByPartitionAndB
partitionColIndiceVec.push_back(safeArray.elems()[i]);
}
- MemoryManager* memoryManager = reinterpret_cast<MemoryManager*>(memoryManagerId);
auto result = batch->toUnsafeRow(0);
auto rowBytes = result.data();
- auto newBatchHandle = ctx->objectStore()->save(ctx->select(memoryManager, batch, partitionColIndiceVec));
+ auto newBatchHandle = ctx->saveObject(ctx->select(batch, partitionColIndiceVec));
auto bytesSize = result.size();
jbyteArray bytesArray = env->NewByteArray(bytesSize);
@@ -1158,138 +1200,12 @@ Java_org_apache_gluten_datasource_DatasourceJniWrapper_splitBlockByPartitionAndB
JNI_METHOD_END(nullptr)
}
-JNIEXPORT jlong JNICALL Java_org_apache_gluten_memory_alloc_NativeMemoryAllocator_getAllocator( // NOLINT
- JNIEnv* env,
- jclass,
- jstring jTypeName) {
- JNI_METHOD_START
- std::string typeName = jStringToCString(env, jTypeName);
- std::shared_ptr<MemoryAllocator>* allocator = new std::shared_ptr<MemoryAllocator>;
- if (typeName == "DEFAULT") {
- *allocator = defaultMemoryAllocator();
- } else {
- delete allocator;
- allocator = nullptr;
- throw GlutenException("Unexpected allocator type name: " + typeName);
- }
- return reinterpret_cast<jlong>(allocator);
- JNI_METHOD_END(kInvalidResourceHandle)
-}
-
-JNIEXPORT void JNICALL Java_org_apache_gluten_memory_alloc_NativeMemoryAllocator_releaseAllocator( // NOLINT
- JNIEnv* env,
- jclass,
- jlong allocatorId) {
- JNI_METHOD_START
- delete reinterpret_cast<std::shared_ptr<MemoryAllocator>*>(allocatorId);
- JNI_METHOD_END()
-}
-
-JNIEXPORT jlong JNICALL Java_org_apache_gluten_memory_alloc_NativeMemoryAllocator_bytesAllocated( // NOLINT
- JNIEnv* env,
- jclass,
- jlong allocatorId) {
- JNI_METHOD_START
- auto* alloc = reinterpret_cast<std::shared_ptr<MemoryAllocator>*>(allocatorId);
- if (alloc == nullptr) {
- throw gluten::GlutenException("Memory allocator instance not found. It may not exist nor has been closed");
- }
- return (*alloc)->getBytes();
- JNI_METHOD_END(kInvalidResourceHandle)
-}
-
-JNIEXPORT jlong JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_create( // NOLINT
- JNIEnv* env,
- jclass,
- jstring jbackendType,
- jstring jnmmName,
- jlong allocatorId,
- jobject jlistener) {
- JNI_METHOD_START
- JavaVM* vm;
- if (env->GetJavaVM(&vm) != JNI_OK) {
- throw gluten::GlutenException("Unable to get JavaVM instance");
- }
- auto allocator = reinterpret_cast<std::shared_ptr<MemoryAllocator>*>(allocatorId);
- if (allocator == nullptr) {
- throw gluten::GlutenException("Allocator does not exist or has been closed");
- }
-
- std::unique_ptr<AllocationListener> listener =
- std::make_unique<SparkAllocationListener>(vm, jlistener, reserveMemoryMethod, unreserveMemoryMethod);
-
- if (gluten::backtrace_allocation) {
- listener = std::make_unique<BacktraceAllocationListener>(std::move(listener));
- }
-
- auto name = jStringToCString(env, jnmmName);
- auto backendType = jStringToCString(env, jbackendType);
- // TODO: move memory manager into Runtime then we can use more general Runtime.
- auto runtime = gluten::Runtime::create(backendType);
- auto manager = runtime->createMemoryManager(name, *allocator, std::move(listener));
- gluten::Runtime::release(runtime);
- return reinterpret_cast<jlong>(manager);
- JNI_METHOD_END(kInvalidResourceHandle)
-}
-
-JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_collectMemoryUsage( // NOLINT
- JNIEnv* env,
- jclass,
- jlong memoryManagerHandle) {
- JNI_METHOD_START
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
-
- const MemoryUsageStats& stats = memoryManager->collectMemoryUsageStats();
- auto size = stats.ByteSizeLong();
- jbyteArray out = env->NewByteArray(size);
- uint8_t buffer[size];
- GLUTEN_CHECK(
- stats.SerializeToArray(reinterpret_cast<void*>(buffer), size),
- "Serialization failed when collecting memory usage stats");
- env->SetByteArrayRegion(out, 0, size, reinterpret_cast<jbyte*>(buffer));
- return out;
- JNI_METHOD_END(nullptr)
-}
-
-JNIEXPORT jlong JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_shrink( // NOLINT
- JNIEnv* env,
- jclass,
- jlong memoryManagerHandle,
- jlong size) {
- JNI_METHOD_START
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
- return memoryManager->shrink(static_cast<int64_t>(size));
- JNI_METHOD_END(kInvalidResourceHandle)
-}
-
-JNIEXPORT void JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_hold( // NOLINT
- JNIEnv* env,
- jclass,
- jlong memoryManagerHandle) {
- JNI_METHOD_START
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
- memoryManager->hold();
- JNI_METHOD_END()
-}
-
-JNIEXPORT void JNICALL Java_org_apache_gluten_memory_nmm_NativeMemoryManager_release( // NOLINT
- JNIEnv* env,
- jclass,
- jlong memoryManagerHandle) {
- JNI_METHOD_START
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
- delete memoryManager;
- JNI_METHOD_END()
-}
-
JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerializerJniWrapper_serialize( // NOLINT
JNIEnv* env,
jobject wrapper,
- jlongArray handles,
- jlong memoryManagerHandle) {
+ jlongArray handles) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
int32_t numBatches = env->GetArrayLength(handles);
auto safeArray = gluten::getLongArrayElementsSafe(env, handles);
@@ -1297,15 +1213,14 @@ JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSeriali
std::vector<std::shared_ptr<ColumnarBatch>> batches;
int64_t numRows = 0L;
for (int32_t i = 0; i < numBatches; i++) {
- auto batch = ctx->objectStore()->retrieve<ColumnarBatch>(safeArray.elems()[i]);
+ auto batch = ObjectStore::retrieve<ColumnarBatch>(safeArray.elems()[i]);
GLUTEN_DCHECK(
batch != nullptr, "Cannot find the ColumnarBatch with handle " + std::to_string(safeArray.elems()[i]));
numRows += batch->numRows();
batches.emplace_back(batch);
}
- auto arrowPool = memoryManager->getArrowMemoryPool();
- auto serializer = ctx->createColumnarBatchSerializer(memoryManager, arrowPool, nullptr);
+ auto serializer = ctx->createColumnarBatchSerializer(nullptr);
auto buffer = serializer->serializeColumnarBatches(batches);
auto bufferArr = env->NewByteArray(buffer->size());
env->SetByteArrayRegion(bufferArr, 0, buffer->size(), reinterpret_cast<const jbyte*>(buffer->data()));
@@ -1320,16 +1235,11 @@ JNIEXPORT jobject JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSeriali
JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerializerJniWrapper_init( // NOLINT
JNIEnv* env,
jobject wrapper,
- jlong cSchema,
- jlong memoryManagerHandle) {
+ jlong cSchema) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto memoryManager = jniCastOrThrow<MemoryManager>(memoryManagerHandle);
-
- auto arrowPool = memoryManager->getArrowMemoryPool();
- return ctx->objectStore()->save(
- ctx->createColumnarBatchSerializer(memoryManager, arrowPool, reinterpret_cast<struct ArrowSchema*>(cSchema)));
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(ctx->createColumnarBatchSerializer(reinterpret_cast<struct ArrowSchema*>(cSchema)));
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerializerJniWrapper_deserialize( // NOLINT
@@ -1340,13 +1250,13 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerialize
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto serializer = ctx->objectStore()->retrieve<ColumnarBatchSerializer>(serializerHandle);
+ auto serializer = ObjectStore::retrieve<ColumnarBatchSerializer>(serializerHandle);
GLUTEN_DCHECK(serializer != nullptr, "ColumnarBatchSerializer cannot be null");
int32_t size = env->GetArrayLength(data);
auto safeArray = gluten::getByteArrayElementsSafe(env, data);
auto batch = serializer->deserialize(safeArray.elems(), size);
- return ctx->objectStore()->save(batch);
- JNI_METHOD_END(kInvalidResourceHandle)
+ return ctx->saveObject(batch);
+ JNI_METHOD_END(kInvalidObjectHandle)
}
JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerializerJniWrapper_close( // NOLINT
@@ -1356,7 +1266,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_vectorized_ColumnarBatchSerializer
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- ctx->objectStore()->release(serializerHandle);
+ ObjectStore::release(serializerHandle);
JNI_METHOD_END()
}
diff --git a/cpp/core/tests/CMakeLists.txt b/cpp/core/tests/CMakeLists.txt
index d8efd9d40..a0f3406f5 100644
--- a/cpp/core/tests/CMakeLists.txt
+++ b/cpp/core/tests/CMakeLists.txt
@@ -18,3 +18,4 @@ if(ENABLE_HBM)
endif()
add_test_case(round_robin_partitioner_test SOURCES RoundRobinPartitionerTest.cc)
+add_test_case(objectstore__test SOURCES ObjectStoreTest.cc)
diff --git a/cpp/core/tests/ObjectStoreTest.cc b/cpp/core/tests/ObjectStoreTest.cc
new file mode 100644
index 000000000..cb5ab3a87
--- /dev/null
+++ b/cpp/core/tests/ObjectStoreTest.cc
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "utils/ObjectStore.h"
+#include <gtest/gtest.h>
+
+using namespace gluten;
+
+TEST(ObjectStore, retreive) {
+ auto store = ObjectStore::create();
+ auto obj = std::make_shared<int32_t>(1);
+ auto handle = store->save(obj);
+ auto retrieved = ObjectStore::retrieve<int32_t>(handle);
+ ASSERT_EQ(*retrieved, 1);
+}
+
+TEST(ObjectStore, retreiveMultiple) {
+ auto store = ObjectStore::create();
+ auto obj1 = std::make_shared<int32_t>(50);
+ auto obj2 = std::make_shared<int32_t>(100);
+ auto handle1 = store->save(obj1);
+ auto handle2 = store->save(obj2);
+ auto retrieved1 = ObjectStore::retrieve<int32_t>(handle1);
+ auto retrieved2 = ObjectStore::retrieve<int32_t>(handle2);
+ ASSERT_EQ(*retrieved1, *obj1);
+ ASSERT_EQ(*retrieved2, *obj2);
+}
+
+TEST(ObjectStore, release) {
+ ObjectHandle handle = kInvalidObjectHandle;
+ auto store = ObjectStore::create();
+ {
+ auto obj = std::make_shared<int32_t>(1);
+ handle = store->save(obj);
+ }
+ auto retrieved = ObjectStore::retrieve<int32_t>(handle);
+ ASSERT_EQ(*retrieved, 1);
+ ObjectStore::release(handle);
+ ASSERT_ANY_THROW(ObjectStore::retrieve<int32_t>(handle));
+}
+
+TEST(ObjectStore, releaseMultiple) {
+ ObjectHandle handle1 = kInvalidObjectHandle;
+ ObjectHandle handle2 = kInvalidObjectHandle;
+ auto store = ObjectStore::create();
+ {
+ auto obj1 = std::make_shared<int32_t>(50);
+ auto obj2 = std::make_shared<int32_t>(100);
+ handle1 = store->save(obj1);
+ handle2 = store->save(obj2);
+ }
+ ASSERT_EQ(*ObjectStore::retrieve<int32_t>(handle1), 50);
+ ASSERT_EQ(*ObjectStore::retrieve<int32_t>(handle2), 100);
+ ObjectStore::release(handle2);
+ ASSERT_EQ(*ObjectStore::retrieve<int32_t>(handle1), 50);
+ ASSERT_ANY_THROW(ObjectStore::retrieve<int32_t>(handle2));
+ ObjectStore::release(handle1);
+ ASSERT_ANY_THROW(ObjectStore::retrieve<int32_t>(handle1));
+ ASSERT_ANY_THROW(ObjectStore::retrieve<int32_t>(handle2));
+}
+
+TEST(ObjectStore, releaseObjectsInMultipleStores) {
+ ObjectHandle handle1 = kInvalidObjectHandle;
+ ObjectHandle handle2 = kInvalidObjectHandle;
+ auto store1 = ObjectStore::create();
+ auto store2 = ObjectStore::create();
+ {
+ auto obj1 = std::make_shared<int32_t>(50);
+ auto obj2 = std::make_shared<int32_t>(100);
+ handle1 = store1->save(obj1);
+ handle2 = store2->save(obj2);
+ }
+ ASSERT_EQ(*ObjectStore::retrieve<int32_t>(handle1), 50);
+ ASSERT_EQ(*ObjectStore::retrieve<int32_t>(handle2), 100);
+ ObjectStore::release(handle2);
+ ASSERT_EQ(*ObjectStore::retrieve<int32_t>(handle1), 50);
+ ASSERT_ANY_THROW(ObjectStore::retrieve<int32_t>(handle2));
+ ObjectStore::release(handle1);
+ ASSERT_ANY_THROW(ObjectStore::retrieve<int32_t>(handle1));
+ ASSERT_ANY_THROW(ObjectStore::retrieve<int32_t>(handle2));
+}
+
+TEST(ObjectStore, releaseMultipleStores) {
+ ObjectHandle handle1 = kInvalidObjectHandle;
+ ObjectHandle handle2 = kInvalidObjectHandle;
+ auto store1 = ObjectStore::create();
+ auto store2 = ObjectStore::create();
+ {
+ auto obj1 = std::make_shared<int32_t>(50);
+ auto obj2 = std::make_shared<int32_t>(100);
+ handle1 = store1->save(obj1);
+ handle2 = store2->save(obj2);
+ }
+ ASSERT_EQ(*ObjectStore::retrieve<int32_t>(handle1), 50);
+ ASSERT_EQ(*ObjectStore::retrieve<int32_t>(handle2), 100);
+ store2.reset();
+ ASSERT_EQ(*ObjectStore::retrieve<int32_t>(handle1), 50);
+ ASSERT_ANY_THROW(ObjectStore::retrieve<int32_t>(handle2));
+ store1.reset();
+ ASSERT_ANY_THROW(ObjectStore::retrieve<int32_t>(handle1));
+ ASSERT_ANY_THROW(ObjectStore::retrieve<int32_t>(handle2));
+}
diff --git a/cpp/core/utils/ObjectStore.cc b/cpp/core/utils/ObjectStore.cc
index 5dc9c7843..648163e4b 100644
--- a/cpp/core/utils/ObjectStore.cc
+++ b/cpp/core/utils/ObjectStore.cc
@@ -22,24 +22,22 @@
gluten::ObjectStore::~ObjectStore() {
// destructing in reversed order (the last added object destructed first)
const std::lock_guard<std::mutex> lock(mtx_);
- for (auto itr = aliveObjectHandles_.rbegin(); itr != aliveObjectHandles_.rend(); itr++) {
+ for (auto itr = aliveObjects_.rbegin(); itr != aliveObjects_.rend(); itr++) {
ResourceHandle handle = *itr;
- if (store_.lookup(handle) == nullptr) {
- LOG(WARNING) << "Fatal: resource handle " + std::to_string(handle) + " not found in store.";
- }
store_.erase(handle);
}
+ stores().erase(storeId_);
}
-gluten::ResourceHandle gluten::ObjectStore::save(std::shared_ptr<void> obj) {
+gluten::ObjectHandle gluten::ObjectStore::save(std::shared_ptr<void> obj) {
const std::lock_guard<std::mutex> lock(mtx_);
ResourceHandle handle = store_.insert(std::move(obj));
- aliveObjectHandles_.insert(handle);
- return handle;
+ aliveObjects_.insert(handle);
+ return toObjHandle(handle);
}
-void gluten::ObjectStore::release(gluten::ResourceHandle handle) {
+void gluten::ObjectStore::release0(gluten::ResourceHandle handle) {
const std::lock_guard<std::mutex> lock(mtx_);
store_.erase(handle);
- aliveObjectHandles_.erase(handle);
+ aliveObjects_.erase(handle);
}
diff --git a/cpp/core/utils/ObjectStore.h b/cpp/core/utils/ObjectStore.h
index 6bfecf599..8e5dd250a 100644
--- a/cpp/core/utils/ObjectStore.h
+++ b/cpp/core/utils/ObjectStore.h
@@ -19,25 +19,73 @@
#include <set>
#include "utils/ResourceMap.h"
-#include "utils/exception.h"
namespace gluten {
+// ObjectHandle is a signed int64 consisting of:
+// 1. 1 - 32 bits is a signed int32 as the object store's ID;
+// 2. 1st bit is always zero to be compatible with jlong;
+// 3. 33 - 64 bits is an unsigned int32 as the object's ID;
+//
+// When the object is tended to be retrieved with its ObjectHandle,
+// the program first finds its resident object store, then looks up
+// for the object in the store.
+using StoreHandle = int32_t;
+using ObjectHandle = int64_t;
+constexpr static ObjectHandle kInvalidObjectHandle = -1;
+
// A store for caching shared-ptrs and enlarging lifecycles of the ptrs to match lifecycle of the store itself by
// default, and also serving release calls to release a ptr in advance. This is typically used in JNI scenario to bind
// a shared-ptr's lifecycle to a Java-side object or some kind of resource manager.
class ObjectStore {
public:
static std::unique_ptr<ObjectStore> create() {
- return std::unique_ptr<ObjectStore>(new ObjectStore());
+ static std::mutex mtx;
+ std::lock_guard<std::mutex> lock(mtx);
+ StoreHandle nextId = stores().nextId();
+ auto store = std::unique_ptr<ObjectStore>(new ObjectStore(nextId));
+ StoreHandle storeId = safeCast<StoreHandle>(stores().insert(store.get()));
+ GLUTEN_CHECK(storeId == nextId, "Store ID mismatched, this should not happen");
+ return store;
+ }
+
+ static void release(ObjectHandle handle) {
+ ResourceHandle storeId = safeCast<ResourceHandle>(handle >> (sizeof(ResourceHandle) * 8));
+ ResourceHandle resourceId = safeCast<ResourceHandle>(handle & std::numeric_limits<ResourceHandle>::max());
+ auto store = stores().lookup(storeId);
+ store->release0(resourceId);
+ }
+
+ template <typename T>
+ static std::shared_ptr<T> retrieve(ObjectHandle handle) {
+ ResourceHandle storeId = safeCast<ResourceHandle>(handle >> (sizeof(ResourceHandle) * 8));
+ ResourceHandle resourceId = safeCast<ResourceHandle>(handle & std::numeric_limits<ResourceHandle>::max());
+ auto store = stores().lookup(storeId);
+ return store->retrieve0<T>(resourceId);
}
virtual ~ObjectStore();
- ResourceHandle save(std::shared_ptr<void> obj);
+ StoreHandle id() {
+ return storeId_;
+ }
+
+ ObjectHandle save(std::shared_ptr<void> obj);
+
+ private:
+ static ResourceMap<ObjectStore*>& stores() {
+ static ResourceMap<ObjectStore*> stores;
+ return stores;
+ }
+
+ ObjectHandle toObjHandle(ResourceHandle rh) {
+ ObjectHandle prefix = static_cast<ObjectHandle>(storeId_) << (sizeof(ResourceHandle) * 8);
+ ObjectHandle objHandle = prefix + rh;
+ return objHandle;
+ }
template <typename T>
- std::shared_ptr<T> retrieve(ResourceHandle handle) {
+ std::shared_ptr<T> retrieve0(ResourceHandle handle) {
const std::lock_guard<std::mutex> lock(mtx_);
std::shared_ptr<void> object = store_.lookup(handle);
// Programming carefully. This will lead to ub if wrong typename T was passed in.
@@ -45,12 +93,12 @@ class ObjectStore {
return casted;
}
- void release(ResourceHandle handle);
+ void release0(ResourceHandle handle);
- private:
- ObjectStore(){};
+ ObjectStore(StoreHandle storeId) : storeId_(storeId){};
+ StoreHandle storeId_;
ResourceMap<std::shared_ptr<void>> store_;
- std::set<ResourceHandle> aliveObjectHandles_;
+ std::set<ResourceHandle> aliveObjects_;
std::mutex mtx_;
};
} // namespace gluten
diff --git a/cpp/core/utils/ResourceMap.h b/cpp/core/utils/ResourceMap.h
index c47fbd9ec..580ad4f6f 100644
--- a/cpp/core/utils/ResourceMap.h
+++ b/cpp/core/utils/ResourceMap.h
@@ -17,13 +17,25 @@
#pragma once
+#include <atomic>
+#include <limits>
#include <mutex>
#include <unordered_map>
+#include "utils/exception.h"
namespace gluten {
+using ResourceHandle = uint32_t;
+static_assert(std::numeric_limits<ResourceHandle>::min() == 0);
-using ResourceHandle = int64_t;
-constexpr static ResourceHandle kInvalidResourceHandle = -1;
+template <typename T, typename F>
+T safeCast(F f) {
+ GLUTEN_CHECK(sizeof(T) <= sizeof(F), "Vain safe casting");
+ F min = 0;
+ F max = static_cast<F>(std::numeric_limits<T>::max());
+ GLUTEN_CHECK(f >= min, "Safe casting a negative number");
+ GLUTEN_CHECK(f <= max, "Number overflow");
+ return static_cast<T>(f);
+}
/**
* An utility class that map resource handle to its shared pointers.
@@ -36,40 +48,48 @@ class ResourceMap {
ResourceMap() : resourceId_(kInitResourceId) {}
ResourceHandle insert(TResource holder) {
- ResourceHandle result = resourceId_++;
+ ResourceHandle result = safeCast<ResourceHandle>(resourceId_++);
+ const std::lock_guard<std::mutex> lock(mtx_);
map_.insert(std::pair<ResourceHandle, TResource>(result, holder));
return result;
}
void erase(ResourceHandle moduleId) {
- map_.erase(moduleId);
+ const std::lock_guard<std::mutex> lock(mtx_);
+ GLUTEN_CHECK(map_.erase(moduleId) == 1, "Module not found in resource map: " + std::to_string(moduleId));
}
TResource lookup(ResourceHandle moduleId) {
+ const std::lock_guard<std::mutex> lock(mtx_);
auto it = map_.find(moduleId);
- if (it != map_.end()) {
- return it->second;
- }
- return nullptr;
+ GLUTEN_CHECK(it != map_.end(), "Module not found in resource map: " + std::to_string(moduleId));
+ return it->second;
}
void clear() {
+ const std::lock_guard<std::mutex> lock(mtx_);
map_.clear();
}
size_t size() {
+ const std::lock_guard<std::mutex> lock(mtx_);
return map_.size();
}
+ size_t nextId() {
+ return resourceId_;
+ }
+
private:
// Initialize the resource id starting value to a number greater than zero
// to allow for easier debugging of uninitialized java variables.
- static constexpr int kInitResourceId = 4;
+ static constexpr size_t kInitResourceId = 4;
- ResourceHandle resourceId_;
+ std::atomic<size_t> resourceId_{0};
// map from resource ids returned to Java and resource pointers
std::unordered_map<ResourceHandle, TResource> map_;
+ std::mutex mtx_;
};
} // namespace gluten
diff --git a/cpp/velox/benchmarks/GenericBenchmark.cc b/cpp/velox/benchmarks/GenericBenchmark.cc
index d8c8c0c24..87d77b715 100644
--- a/cpp/velox/benchmarks/GenericBenchmark.cc
+++ b/cpp/velox/benchmarks/GenericBenchmark.cc
@@ -98,11 +98,8 @@ void setUpBenchmark(::benchmark::internal::Benchmark* bm) {
}
}
-std::shared_ptr<VeloxShuffleWriter> createShuffleWriter(
- Runtime* runtime,
- VeloxMemoryManager* memoryManager,
- const std::string& dataFile,
- const std::vector<std::string>& localDirs) {
+std::shared_ptr<VeloxShuffleWriter>
+createShuffleWriter(Runtime* runtime, const std::string& dataFile, const std::vector<std::string>& localDirs) {
PartitionWriterOptions partitionWriterOptions{};
// Configure compression.
@@ -131,13 +128,13 @@ std::shared_ptr<VeloxShuffleWriter> createShuffleWriter(
partitionWriter = std::make_unique<RssPartitionWriter>(
FLAGS_shuffle_partitions,
std::move(partitionWriterOptions),
- memoryManager->getArrowMemoryPool(),
+ runtime->memoryManager()->getArrowMemoryPool(),
std::move(rssClient));
} else {
partitionWriter = std::make_unique<LocalPartitionWriter>(
FLAGS_shuffle_partitions,
std::move(partitionWriterOptions),
- memoryManager->getArrowMemoryPool(),
+ runtime->memoryManager()->getArrowMemoryPool(),
dataFile,
localDirs);
}
@@ -147,8 +144,8 @@ std::shared_ptr<VeloxShuffleWriter> createShuffleWriter(
if (FLAGS_shuffle_writer == "sort") {
options.shuffleWriterType = gluten::kSortShuffle;
}
- auto shuffleWriter = runtime->createShuffleWriter(
- FLAGS_shuffle_partitions, std::move(partitionWriter), std::move(options), memoryManager);
+ auto shuffleWriter =
+ runtime->createShuffleWriter(FLAGS_shuffle_partitions, std::move(partitionWriter), std::move(options));
return std::reinterpret_pointer_cast<VeloxShuffleWriter>(shuffleWriter);
}
@@ -178,7 +175,6 @@ void setCpu(::benchmark::State& state) {
void runShuffle(
Runtime* runtime,
- VeloxMemoryManager* memoryManager,
BenchmarkAllocationListener* listener,
const std::shared_ptr<gluten::ResultIterator>& resultIter,
WriterMetrics& metrics) {
@@ -187,7 +183,7 @@ void runShuffle(
bool isFromEnv;
GLUTEN_THROW_NOT_OK(setLocalDirsAndDataFileFromEnv(dataFile, localDirs, isFromEnv));
- auto shuffleWriter = createShuffleWriter(runtime, memoryManager, dataFile, localDirs);
+ auto shuffleWriter = createShuffleWriter(runtime, dataFile, localDirs);
listener->setShuffleWriter(shuffleWriter.get());
int64_t totalTime = 0;
@@ -226,18 +222,19 @@ void updateBenchmarkMetrics(
} // namespace
+using RuntimeFactory = std::function<VeloxRuntime*(std::unique_ptr<AllocationListener> listener)>;
+
auto BM_Generic = [](::benchmark::State& state,
const std::string& planFile,
const std::vector<std::string>& splitFiles,
const std::vector<std::string>& dataFiles,
- Runtime* runtime,
+ RuntimeFactory runtimeFactory,
FileReaderType readerType) {
setCpu(state);
auto listener = std::make_unique<BenchmarkAllocationListener>(FLAGS_memory_limit);
auto* listenerPtr = listener.get();
- auto memoryManager = std::make_unique<gluten::VeloxMemoryManager>(
- "generic_benchmark", gluten::defaultMemoryAllocator(), std::move(listener));
+ auto runtime = runtimeFactory(std::move(listener));
auto plan = getPlanFromFile("Plan", planFile);
std::vector<std::string> splits{};
@@ -271,18 +268,17 @@ auto BM_Generic = [](::benchmark::State& state,
for (auto& split : splits) {
runtime->parseSplitInfo(reinterpret_cast<uint8_t*>(split.data()), split.size(), std::nullopt);
}
- auto resultIter = runtime->createResultIterator(
- memoryManager.get(), "/tmp/test-spill", std::move(inputIters), runtime->getConfMap());
+ auto resultIter = runtime->createResultIterator("/tmp/test-spill", std::move(inputIters), runtime->getConfMap());
listenerPtr->setIterator(resultIter.get());
if (FLAGS_with_shuffle) {
- runShuffle(runtime, memoryManager.get(), listenerPtr, resultIter, writerMetrics);
+ runShuffle(runtime, listenerPtr, resultIter, writerMetrics);
} else {
// May write the output into file.
auto veloxPlan = dynamic_cast<gluten::VeloxRuntime*>(runtime)->getVeloxPlan();
ArrowSchema cSchema;
- toArrowSchema(veloxPlan->outputType(), memoryManager->getLeafMemoryPool().get(), &cSchema);
+ toArrowSchema(veloxPlan->outputType(), runtime->memoryManager()->getLeafMemoryPool().get(), &cSchema);
GLUTEN_ASSIGN_OR_THROW(auto outputSchema, arrow::ImportSchema(&cSchema));
ArrowWriter writer{FLAGS_save_output};
state.PauseTiming();
@@ -328,33 +324,36 @@ auto BM_Generic = [](::benchmark::State& state,
}
updateBenchmarkMetrics(state, elapsedTime, readInputTime, writerMetrics);
+ Runtime::release(runtime);
};
-auto BM_ShuffleWrite =
- [](::benchmark::State& state, const std::string& inputFile, Runtime* runtime, FileReaderType readerType) {
- setCpu(state);
-
- auto listener = std::make_unique<BenchmarkAllocationListener>(FLAGS_memory_limit);
- auto* listenerPtr = listener.get();
- auto memoryManager = std::make_unique<gluten::VeloxMemoryManager>(
- "generic_benchmark", gluten::defaultMemoryAllocator(), std::move(listener));
-
- WriterMetrics writerMetrics{};
- int64_t readInputTime = 0;
- int64_t elapsedTime = 0;
- {
- ScopedTimer timer(&elapsedTime);
- for (auto _ : state) {
- auto resultIter = getInputIteratorFromFileReader(inputFile, readerType);
- runShuffle(runtime, memoryManager.get(), listenerPtr, resultIter, writerMetrics);
-
- auto reader = static_cast<FileReaderIterator*>(resultIter->getInputIter());
- readInputTime += reader->getCollectBatchTime();
- }
- }
+auto BM_ShuffleWrite = [](::benchmark::State& state,
+ const std::string& inputFile,
+ RuntimeFactory runtimeFactory,
+ FileReaderType readerType) {
+ setCpu(state);
- updateBenchmarkMetrics(state, elapsedTime, readInputTime, writerMetrics);
- };
+ auto listener = std::make_unique<BenchmarkAllocationListener>(FLAGS_memory_limit);
+ auto* listenerPtr = listener.get();
+ auto runtime = runtimeFactory(std::move(listener));
+
+ WriterMetrics writerMetrics{};
+ int64_t readInputTime = 0;
+ int64_t elapsedTime = 0;
+ {
+ ScopedTimer timer(&elapsedTime);
+ for (auto _ : state) {
+ auto resultIter = getInputIteratorFromFileReader(inputFile, readerType);
+ runShuffle(runtime, listenerPtr, resultIter, writerMetrics);
+
+ auto reader = static_cast<FileReaderIterator*>(resultIter->getInputIter());
+ readInputTime += reader->getCollectBatchTime();
+ }
+ }
+
+ updateBenchmarkMetrics(state, elapsedTime, readInputTime, writerMetrics);
+ Runtime::release(runtime);
+};
int main(int argc, char** argv) {
::benchmark::Initialize(&argc, argv);
@@ -512,22 +511,26 @@ int main(int argc, char** argv) {
}
}
- auto runtime = Runtime::create(kVeloxRuntimeKind, sessionConf);
+ RuntimeFactory runtimeFactory = [=](std::unique_ptr<AllocationListener> listener) {
+ return dynamic_cast<VeloxRuntime*>(Runtime::create(kVeloxRuntimeKind, std::move(listener), sessionConf));
+ };
#define GENERIC_BENCHMARK(READER_TYPE) \
do { \
- auto* bm = ::benchmark::RegisterBenchmark( \
- "GenericBenchmark", BM_Generic, substraitJsonFile, splitFiles, dataFiles, runtime, READER_TYPE) \
- ->MeasureProcessCPUTime() \
- ->UseRealTime(); \
+ auto* bm = \
+ ::benchmark::RegisterBenchmark( \
+ "GenericBenchmark", BM_Generic, substraitJsonFile, splitFiles, dataFiles, runtimeFactory, READER_TYPE) \
+ ->MeasureProcessCPUTime() \
+ ->UseRealTime(); \
setUpBenchmark(bm); \
} while (0)
#define SHUFFLE_WRITE_BENCHMARK(READER_TYPE) \
do { \
- auto* bm = ::benchmark::RegisterBenchmark("ShuffleWrite", BM_ShuffleWrite, dataFiles[0], runtime, READER_TYPE) \
- ->MeasureProcessCPUTime() \
- ->UseRealTime(); \
+ auto* bm = \
+ ::benchmark::RegisterBenchmark("ShuffleWrite", BM_ShuffleWrite, dataFiles[0], runtimeFactory, READER_TYPE) \
+ ->MeasureProcessCPUTime() \
+ ->UseRealTime(); \
setUpBenchmark(bm); \
} while (0)
@@ -561,7 +564,6 @@ int main(int argc, char** argv) {
::benchmark::RunSpecifiedBenchmarks();
::benchmark::Shutdown();
- Runtime::release(runtime);
gluten::VeloxBackend::get()->tearDown();
return 0;
diff --git a/cpp/velox/benchmarks/ParquetWriteBenchmark.cc b/cpp/velox/benchmarks/ParquetWriteBenchmark.cc
index 894c35351..45348ed4a 100644
--- a/cpp/velox/benchmarks/ParquetWriteBenchmark.cc
+++ b/cpp/velox/benchmarks/ParquetWriteBenchmark.cc
@@ -257,7 +257,7 @@ class GoogleBenchmarkVeloxParquetWriteCacheScanBenchmark : public GoogleBenchmar
// reuse the ParquetWriteConverter for batches caused system % increase a lot
auto fileName = "velox_parquet_write.parquet";
- auto runtime = Runtime::create(kVeloxRuntimeKind);
+ auto runtime = Runtime::create(kVeloxRuntimeKind, AllocationListener::noop());
auto memoryManager = getDefaultMemoryManager();
auto veloxPool = memoryManager->getAggregateMemoryPool();
diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc
index 1ec587996..a3658faa3 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -55,8 +55,10 @@ using namespace facebook;
namespace gluten {
namespace {
-gluten::Runtime* veloxRuntimeFactory(const std::unordered_map<std::string, std::string>& sessionConf) {
- return new gluten::VeloxRuntime(sessionConf);
+gluten::Runtime* veloxRuntimeFactory(
+ std::unique_ptr<AllocationListener> listener,
+ const std::unordered_map<std::string, std::string>& sessionConf) {
+ return new gluten::VeloxRuntime(std::move(listener), sessionConf);
}
} // namespace
diff --git a/cpp/velox/compute/VeloxRuntime.cc b/cpp/velox/compute/VeloxRuntime.cc
index 738ce99a3..fde87b5b4 100644
--- a/cpp/velox/compute/VeloxRuntime.cc
+++ b/cpp/velox/compute/VeloxRuntime.cc
@@ -56,8 +56,12 @@ using namespace facebook;
namespace gluten {
-VeloxRuntime::VeloxRuntime(const std::unordered_map<std::string, std::string>& confMap) : Runtime(confMap) {
+VeloxRuntime::VeloxRuntime(
+ std::unique_ptr<AllocationListener> listener,
+ const std::unordered_map<std::string, std::string>& confMap)
+ : Runtime(std::make_shared<VeloxMemoryManager>(std::move(listener)), confMap) {
// Refresh session config.
+ vmm_ = dynamic_cast<VeloxMemoryManager*>(memoryManager_.get());
veloxCfg_ = std::make_shared<facebook::velox::core::MemConfig>(confMap_);
debugModeEnabled_ = veloxCfg_->get<bool>(kDebugModeEnabled, false);
FLAGS_minloglevel = veloxCfg_->get<uint32_t>(kGlogSeverityLevel, FLAGS_minloglevel);
@@ -127,15 +131,17 @@ void VeloxRuntime::injectWriteFilesTempPath(const std::string& path) {
writeFilesTempPath_ = path;
}
+VeloxMemoryManager* VeloxRuntime::memoryManager() {
+ return vmm_;
+}
+
std::shared_ptr<ResultIterator> VeloxRuntime::createResultIterator(
- MemoryManager* memoryManager,
const std::string& spillDir,
const std::vector<std::shared_ptr<ResultIterator>>& inputs,
const std::unordered_map<std::string, std::string>& sessionConf) {
LOG_IF(INFO, debugModeEnabled_) << "VeloxRuntime session config:" << printConfig(confMap_);
- VeloxPlanConverter veloxPlanConverter(
- inputs, getLeafVeloxPool(memoryManager).get(), sessionConf, writeFilesTempPath_);
+ VeloxPlanConverter veloxPlanConverter(inputs, vmm_->getLeafMemoryPool().get(), sessionConf, writeFilesTempPath_);
veloxPlan_ = veloxPlanConverter.toVeloxPlan(substraitPlan_, std::move(localFiles_));
// Scan node can be required.
@@ -146,15 +152,14 @@ std::shared_ptr<ResultIterator> VeloxRuntime::createResultIterator(
// Separate the scan ids and stream ids, and get the scan infos.
getInfoAndIds(veloxPlanConverter.splitInfos(), veloxPlan_->leafPlanNodeIds(), scanInfos, scanIds, streamIds);
- auto* vmm = toVeloxMemoryManager(memoryManager);
auto wholestageIter = std::make_unique<WholeStageResultIterator>(
- vmm, veloxPlan_, scanIds, scanInfos, streamIds, spillDir, sessionConf, taskInfo_);
+ vmm_, veloxPlan_, scanIds, scanInfos, streamIds, spillDir, sessionConf, taskInfo_);
return std::make_shared<ResultIterator>(std::move(wholestageIter), this);
}
-std::shared_ptr<ColumnarToRowConverter> VeloxRuntime::createColumnar2RowConverter(MemoryManager* memoryManager) {
- auto ctxVeloxPool = getLeafVeloxPool(memoryManager);
- return std::make_shared<VeloxColumnarToRowConverter>(ctxVeloxPool);
+std::shared_ptr<ColumnarToRowConverter> VeloxRuntime::createColumnar2RowConverter() {
+ auto veloxPool = vmm_->getLeafMemoryPool();
+ return std::make_shared<VeloxColumnarToRowConverter>(veloxPool);
}
std::shared_ptr<ColumnarBatch> VeloxRuntime::createOrGetEmptySchemaBatch(int32_t numRows) {
@@ -167,51 +172,45 @@ std::shared_ptr<ColumnarBatch> VeloxRuntime::createOrGetEmptySchemaBatch(int32_t
}
std::shared_ptr<ColumnarBatch> VeloxRuntime::select(
- MemoryManager* memoryManager,
std::shared_ptr<ColumnarBatch> batch,
std::vector<int32_t> columnIndices) {
- auto ctxVeloxPool = getLeafVeloxPool(memoryManager);
- auto veloxBatch = gluten::VeloxColumnarBatch::from(ctxVeloxPool.get(), batch);
- auto outputBatch = veloxBatch->select(ctxVeloxPool.get(), std::move(columnIndices));
+ auto veloxPool = vmm_->getLeafMemoryPool();
+ auto veloxBatch = gluten::VeloxColumnarBatch::from(veloxPool.get(), batch);
+ auto outputBatch = veloxBatch->select(veloxPool.get(), std::move(columnIndices));
return outputBatch;
}
-std::shared_ptr<RowToColumnarConverter> VeloxRuntime::createRow2ColumnarConverter(
- MemoryManager* memoryManager,
- struct ArrowSchema* cSchema) {
- auto ctxVeloxPool = getLeafVeloxPool(memoryManager);
- return std::make_shared<VeloxRowToColumnarConverter>(cSchema, ctxVeloxPool);
+std::shared_ptr<RowToColumnarConverter> VeloxRuntime::createRow2ColumnarConverter(struct ArrowSchema* cSchema) {
+ auto veloxPool = vmm_->getLeafMemoryPool();
+ return std::make_shared<VeloxRowToColumnarConverter>(cSchema, veloxPool);
}
std::shared_ptr<ShuffleWriter> VeloxRuntime::createShuffleWriter(
int numPartitions,
std::unique_ptr<PartitionWriter> partitionWriter,
- ShuffleWriterOptions options,
- MemoryManager* memoryManager) {
- auto ctxPool = getLeafVeloxPool(memoryManager);
- auto arrowPool = memoryManager->getArrowMemoryPool();
- std::shared_ptr<ShuffleWriter> shuffleWriter;
+ ShuffleWriterOptions options) {
+ auto veloxPool = vmm_->getLeafMemoryPool();
+ auto arrowPool = vmm_->getArrowMemoryPool();
GLUTEN_ASSIGN_OR_THROW(
- shuffleWriter,
+ std::shared_ptr<ShuffleWriter> shuffleWriter,
VeloxShuffleWriter::create(
options.shuffleWriterType,
numPartitions,
std::move(partitionWriter),
std::move(options),
- ctxPool,
+ veloxPool,
arrowPool));
return shuffleWriter;
}
std::shared_ptr<Datasource> VeloxRuntime::createDatasource(
const std::string& filePath,
- MemoryManager* memoryManager,
std::shared_ptr<arrow::Schema> schema) {
static std::atomic_uint32_t id{0UL};
- auto veloxPool = getAggregateVeloxPool(memoryManager)->addAggregateChild("datasource." + std::to_string(id++));
+ auto veloxPool = vmm_->getAggregateMemoryPool()->addAggregateChild("datasource." + std::to_string(id++));
// Pass a dedicate pool for S3 and GCS sinks as can't share veloxPool
// with parquet writer.
- auto sinkPool = getLeafVeloxPool(memoryManager);
+ auto sinkPool = vmm_->getLeafMemoryPool();
if (isSupportedHDFSPath(filePath)) {
#ifdef ENABLE_HDFS
return std::make_shared<VeloxParquetDatasourceHDFS>(filePath, veloxPool, sinkPool, schema);
@@ -246,12 +245,10 @@ std::shared_ptr<Datasource> VeloxRuntime::createDatasource(
std::shared_ptr<ShuffleReader> VeloxRuntime::createShuffleReader(
std::shared_ptr<arrow::Schema> schema,
- ShuffleReaderOptions options,
- arrow::MemoryPool* pool,
- MemoryManager* memoryManager) {
+ ShuffleReaderOptions options) {
auto rowType = facebook::velox::asRowType(gluten::fromArrowSchema(schema));
auto codec = gluten::createArrowIpcCodec(options.compressionType, options.codecBackend);
- auto ctxVeloxPool = getLeafVeloxPool(memoryManager);
+ auto ctxVeloxPool = vmm_->getLeafMemoryPool();
auto veloxCompressionType = facebook::velox::common::stringToCompressionKind(options.compressionTypeStr);
auto deserializerFactory = std::make_unique<gluten::VeloxColumnarBatchDeserializerFactory>(
schema,
@@ -259,19 +256,17 @@ std::shared_ptr<ShuffleReader> VeloxRuntime::createShuffleReader(
veloxCompressionType,
rowType,
options.batchSize,
- pool,
+ vmm_->getArrowMemoryPool(),
ctxVeloxPool,
options.shuffleWriterType);
auto reader = std::make_shared<VeloxShuffleReader>(std::move(deserializerFactory));
return reader;
}
-std::unique_ptr<ColumnarBatchSerializer> VeloxRuntime::createColumnarBatchSerializer(
- MemoryManager* memoryManager,
- arrow::MemoryPool* arrowPool,
- struct ArrowSchema* cSchema) {
- auto ctxVeloxPool = getLeafVeloxPool(memoryManager);
- return std::make_unique<VeloxColumnarBatchSerializer>(arrowPool, ctxVeloxPool, cSchema);
+std::unique_ptr<ColumnarBatchSerializer> VeloxRuntime::createColumnarBatchSerializer(struct ArrowSchema* cSchema) {
+ auto arrowPool = vmm_->getArrowMemoryPool();
+ auto veloxPool = vmm_->getLeafMemoryPool();
+ return std::make_unique<VeloxColumnarBatchSerializer>(arrowPool, veloxPool, cSchema);
}
void VeloxRuntime::dumpConf(const std::string& path) {
diff --git a/cpp/velox/compute/VeloxRuntime.h b/cpp/velox/compute/VeloxRuntime.h
index 80408bccb..096ecb6fb 100644
--- a/cpp/velox/compute/VeloxRuntime.h
+++ b/cpp/velox/compute/VeloxRuntime.h
@@ -33,81 +33,49 @@ inline static const std::string kVeloxRuntimeKind{"velox"};
class VeloxRuntime final : public Runtime {
public:
- explicit VeloxRuntime(const std::unordered_map<std::string, std::string>& confMap);
+ explicit VeloxRuntime(
+ std::unique_ptr<AllocationListener> listener,
+ const std::unordered_map<std::string, std::string>& confMap);
void parsePlan(const uint8_t* data, int32_t size, std::optional<std::string> dumpFile) override;
void parseSplitInfo(const uint8_t* data, int32_t size, std::optional<std::string> dumpFile) override;
- static std::shared_ptr<facebook::velox::memory::MemoryPool> getAggregateVeloxPool(MemoryManager* memoryManager) {
- return toVeloxMemoryManager(memoryManager)->getAggregateMemoryPool();
- }
-
- static std::shared_ptr<facebook::velox::memory::MemoryPool> getLeafVeloxPool(MemoryManager* memoryManager) {
- return toVeloxMemoryManager(memoryManager)->getLeafMemoryPool();
- }
-
- static VeloxMemoryManager* toVeloxMemoryManager(MemoryManager* memoryManager) {
- if (auto veloxMemoryManager = dynamic_cast<VeloxMemoryManager*>(memoryManager)) {
- return veloxMemoryManager;
- } else {
- GLUTEN_CHECK(false, "Velox memory manager should be used for Velox runtime.");
- }
- }
-
- MemoryManager* createMemoryManager(
- const std::string& name,
- std::shared_ptr<MemoryAllocator> allocator,
- std::unique_ptr<AllocationListener> listener) override {
- return new VeloxMemoryManager(name, allocator, std::move(listener));
- }
+ VeloxMemoryManager* memoryManager() override;
// FIXME This is not thread-safe?
std::shared_ptr<ResultIterator> createResultIterator(
- MemoryManager* memoryManager,
const std::string& spillDir,
const std::vector<std::shared_ptr<ResultIterator>>& inputs = {},
const std::unordered_map<std::string, std::string>& sessionConf = {}) override;
- std::shared_ptr<ColumnarToRowConverter> createColumnar2RowConverter(MemoryManager* memoryManager) override;
+ std::shared_ptr<ColumnarToRowConverter> createColumnar2RowConverter() override;
std::shared_ptr<ColumnarBatch> createOrGetEmptySchemaBatch(int32_t numRows) override;
- std::shared_ptr<ColumnarBatch> select(
- MemoryManager* memoryManager,
- std::shared_ptr<ColumnarBatch> batch,
- std::vector<int32_t> columnIndices) override;
+ std::shared_ptr<ColumnarBatch> select(std::shared_ptr<ColumnarBatch> batch, std::vector<int32_t> columnIndices)
+ override;
- std::shared_ptr<RowToColumnarConverter> createRow2ColumnarConverter(
- MemoryManager* memoryManager,
- struct ArrowSchema* cSchema) override;
+ std::shared_ptr<RowToColumnarConverter> createRow2ColumnarConverter(struct ArrowSchema* cSchema) override;
std::shared_ptr<ShuffleWriter> createShuffleWriter(
int numPartitions,
std::unique_ptr<PartitionWriter> partitionWriter,
- ShuffleWriterOptions options,
- MemoryManager* memoryManager) override;
+ ShuffleWriterOptions options) override;
Metrics* getMetrics(ColumnarBatchIterator* rawIter, int64_t exportNanos) override {
auto iter = static_cast<WholeStageResultIterator*>(rawIter);
return iter->getMetrics(exportNanos);
}
- std::shared_ptr<Datasource> createDatasource(
- const std::string& filePath,
- MemoryManager* memoryManager,
- std::shared_ptr<arrow::Schema> schema) override;
+ std::shared_ptr<Datasource> createDatasource(const std::string& filePath, std::shared_ptr<arrow::Schema> schema)
+ override;
std::shared_ptr<ShuffleReader> createShuffleReader(
std::shared_ptr<arrow::Schema> schema,
- ShuffleReaderOptions options,
- arrow::MemoryPool* pool,
- MemoryManager* memoryManager) override;
+ ShuffleReaderOptions options) override;
- std::unique_ptr<ColumnarBatchSerializer> createColumnarBatchSerializer(
- MemoryManager* memoryManager,
- arrow::MemoryPool* arrowPool,
- struct ArrowSchema* cSchema) override;
+ std::unique_ptr<ColumnarBatchSerializer> createColumnarBatchSerializer(struct ArrowSchema* cSchema) override;
std::string planString(bool details, const std::unordered_map<std::string, std::string>& sessionConf) override;
@@ -131,6 +99,7 @@ class VeloxRuntime final : public Runtime {
std::vector<facebook::velox::core::PlanNodeId>& streamIds);
private:
+ VeloxMemoryManager* vmm_;
std::shared_ptr<const facebook::velox::core::PlanNode> veloxPlan_;
std::shared_ptr<facebook::velox::Config> veloxCfg_;
bool debugModeEnabled_{false};
diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc
index 5920a0898..cbc6c838b 100644
--- a/cpp/velox/compute/WholeStageResultIterator.cc
+++ b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -83,8 +83,13 @@ WholeStageResultIterator::WholeStageResultIterator(
std::unordered_set<velox::core::PlanNodeId> emptySet;
velox::core::PlanFragment planFragment{planNode, velox::core::ExecutionStrategy::kUngrouped, 1, emptySet};
std::shared_ptr<velox::core::QueryCtx> queryCtx = createNewVeloxQueryCtx();
+ static std::atomic<uint32_t> vtId{0}; // Velox task ID to distinguish from Spark task ID.
task_ = velox::exec::Task::create(
- fmt::format("Gluten_Stage_{}_TID_{}", std::to_string(taskInfo_.stageId), std::to_string(taskInfo_.taskId)),
+ fmt::format(
+ "Gluten_Stage_{}_TID_{}_VTID_{}",
+ std::to_string(taskInfo_.stageId),
+ std::to_string(taskInfo_.taskId),
+ std::to_string(vtId++)),
std::move(planFragment),
0,
std::move(queryCtx),
diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc
index 3b52eaa86..f11edf1a4 100644
--- a/cpp/velox/jni/VeloxJniWrapper.cc
+++ b/cpp/velox/jni/VeloxJniWrapper.cc
@@ -30,6 +30,7 @@
#include "jni/JniFileSystem.h"
#include "memory/VeloxMemoryManager.h"
#include "substrait/SubstraitToVeloxPlanValidator.h"
+#include "utils/ObjectStore.h"
#include "utils/VeloxBatchAppender.h"
#include "velox/common/base/BloomFilter.h"
@@ -159,8 +160,8 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_
auto filter = std::make_shared<velox::BloomFilter<std::allocator<uint64_t>>>();
filter->reset(capacity);
GLUTEN_CHECK(filter->isSet(), "Bloom-filter is not initialized");
- return ctx->objectStore()->save(filter);
- JNI_METHOD_END(gluten::kInvalidResourceHandle)
+ return ctx->saveObject(filter);
+ JNI_METHOD_END(gluten::kInvalidObjectHandle)
}
JNIEXPORT jlong JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_init( // NOLINT
@@ -173,8 +174,8 @@ JNIEXPORT jlong JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_
auto filter = std::make_shared<velox::BloomFilter<std::allocator<uint64_t>>>();
uint8_t* serialized = safeArray.elems();
filter->merge(reinterpret_cast<char*>(serialized));
- return ctx->objectStore()->save(filter);
- JNI_METHOD_END(gluten::kInvalidResourceHandle)
+ return ctx->saveObject(filter);
+ JNI_METHOD_END(gluten::kInvalidObjectHandle)
}
JNIEXPORT void JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_insertLong( // NOLINT
@@ -184,7 +185,7 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_i
jlong item) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto filter = ctx->objectStore()->retrieve<velox::BloomFilter<std::allocator<uint64_t>>>(handle);
+ auto filter = gluten::ObjectStore::retrieve<velox::BloomFilter<std::allocator<uint64_t>>>(handle);
GLUTEN_CHECK(filter->isSet(), "Bloom-filter is not initialized");
filter->insert(folly::hasher<int64_t>()(item));
JNI_METHOD_END()
@@ -197,7 +198,7 @@ JNIEXPORT jboolean JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapp
jlong item) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto filter = ctx->objectStore()->retrieve<velox::BloomFilter<std::allocator<uint64_t>>>(handle);
+ auto filter = gluten::ObjectStore::retrieve<velox::BloomFilter<std::allocator<uint64_t>>>(handle);
GLUTEN_CHECK(filter->isSet(), "Bloom-filter is not initialized");
bool out = filter->mayContain(folly::hasher<int64_t>()(item));
return out;
@@ -222,8 +223,8 @@ JNIEXPORT void JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWrapper_m
jlong other) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto to = ctx->objectStore()->retrieve<velox::BloomFilter<std::allocator<uint64_t>>>(handle);
- auto from = ctx->objectStore()->retrieve<velox::BloomFilter<std::allocator<uint64_t>>>(other);
+ auto to = gluten::ObjectStore::retrieve<velox::BloomFilter<std::allocator<uint64_t>>>(handle);
+ auto from = gluten::ObjectStore::retrieve<velox::BloomFilter<std::allocator<uint64_t>>>(other);
GLUTEN_CHECK(to->isSet(), "Bloom-filter is not initialized");
GLUTEN_CHECK(from->isSet(), "Bloom-filter is not initialized");
std::vector<char> serialized = serialize(from.get());
@@ -237,7 +238,7 @@ JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWra
jlong handle) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto filter = ctx->objectStore()->retrieve<velox::BloomFilter<std::allocator<uint64_t>>>(handle);
+ auto filter = gluten::ObjectStore::retrieve<velox::BloomFilter<std::allocator<uint64_t>>>(handle);
GLUTEN_CHECK(filter->isSet(), "Bloom-filter is not initialized");
std::vector<char> buffer = serialize(filter.get());
auto size = buffer.capacity();
@@ -250,18 +251,16 @@ JNIEXPORT jbyteArray JNICALL Java_org_apache_gluten_utils_VeloxBloomFilterJniWra
JNIEXPORT jlong JNICALL Java_org_apache_gluten_utils_VeloxBatchAppenderJniWrapper_create( // NOLINT
JNIEnv* env,
jobject wrapper,
- jlong memoryManagerHandle,
jint minOutputBatchSize,
jobject jIter) {
JNI_METHOD_START
auto ctx = gluten::getRuntime(env, wrapper);
- auto memoryManager = jniCastOrThrow<gluten::MemoryManager>(memoryManagerHandle);
- auto pool = gluten::VeloxRuntime::getLeafVeloxPool(memoryManager);
+ auto pool = dynamic_cast<gluten::VeloxMemoryManager*>(ctx->memoryManager())->getLeafMemoryPool();
auto iter = gluten::makeJniColumnarBatchIterator(env, jIter, ctx, nullptr);
auto appender = std::make_shared<gluten::ResultIterator>(
std::make_unique<gluten::VeloxBatchAppender>(pool.get(), minOutputBatchSize, std::move(iter)));
- return ctx->objectStore()->save(appender);
- JNI_METHOD_END(gluten::kInvalidResourceHandle)
+ return ctx->saveObject(appender);
+ JNI_METHOD_END(gluten::kInvalidObjectHandle)
}
#ifdef __cplusplus
diff --git a/cpp/velox/memory/VeloxMemoryManager.cc b/cpp/velox/memory/VeloxMemoryManager.cc
index efd165b73..442090004 100644
--- a/cpp/velox/memory/VeloxMemoryManager.cc
+++ b/cpp/velox/memory/VeloxMemoryManager.cc
@@ -161,17 +161,14 @@ class ArbitratorFactoryRegister {
gluten::AllocationListener* listener_;
};
-VeloxMemoryManager::VeloxMemoryManager(
- const std::string& name,
- std::shared_ptr<MemoryAllocator> allocator,
- std::unique_ptr<AllocationListener> listener)
- : MemoryManager(), name_(name), listener_(std::move(listener)) {
+VeloxMemoryManager::VeloxMemoryManager(std::unique_ptr<AllocationListener> listener)
+ : MemoryManager(), listener_(std::move(listener)) {
auto reservationBlockSize = VeloxBackend::get()->getBackendConf()->get<uint64_t>(
kMemoryReservationBlockSize, kMemoryReservationBlockSizeDefault);
auto memInitCapacity =
VeloxBackend::get()->getBackendConf()->get<uint64_t>(kVeloxMemInitCapacity, kVeloxMemInitCapacityDefault);
blockListener_ = std::make_unique<BlockAllocationListener>(listener_.get(), reservationBlockSize);
- listenableAlloc_ = std::make_unique<ListenableMemoryAllocator>(allocator.get(), blockListener_.get());
+ listenableAlloc_ = std::make_unique<ListenableMemoryAllocator>(defaultMemoryAllocator().get(), blockListener_.get());
arrowPool_ = std::make_unique<ArrowMemoryPool>(listenableAlloc_.get());
ArbitratorFactoryRegister afr(listener_.get());
@@ -189,11 +186,11 @@ VeloxMemoryManager::VeloxMemoryManager(
veloxMemoryManager_ = std::make_unique<velox::memory::MemoryManager>(mmOptions);
veloxAggregatePool_ = veloxMemoryManager_->addRootPool(
- name_ + "_root",
+ "root",
velox::memory::kMaxMemory, // the 3rd capacity
facebook::velox::memory::MemoryReclaimer::create());
- veloxLeafPool_ = veloxAggregatePool_->addLeafChild(name_ + "_default_leaf");
+ veloxLeafPool_ = veloxAggregatePool_->addLeafChild("default_leaf");
}
namespace {
@@ -317,8 +314,10 @@ bool VeloxMemoryManager::tryDestructSafe() {
VeloxMemoryManager::~VeloxMemoryManager() {
static const uint32_t kWaitTimeoutMs = FLAGS_gluten_velox_aysnc_timeout_on_task_stopping; // 30s by default
uint32_t accumulatedWaitMs = 0UL;
+ bool destructed = false;
for (int32_t tryCount = 0; accumulatedWaitMs < kWaitTimeoutMs; tryCount++) {
- if (tryDestructSafe()) {
+ destructed = tryDestructSafe();
+ if (destructed) {
if (tryCount > 0) {
LOG(INFO) << "All the outstanding memory resources successfully released. ";
}
@@ -330,6 +329,10 @@ VeloxMemoryManager::~VeloxMemoryManager() {
usleep(waitMs * 1000);
accumulatedWaitMs += waitMs;
}
+ if (!destructed) {
+ LOG(ERROR) << "Failed to release Velox memory manager after " << accumulatedWaitMs
+ << "ms as there are still outstanding memory resources. ";
+ }
#ifdef ENABLE_JEMALLOC
je_gluten_malloc_stats_print(NULL, NULL, NULL);
#endif
diff --git a/cpp/velox/memory/VeloxMemoryManager.h b/cpp/velox/memory/VeloxMemoryManager.h
index 3ba5bbf7d..3607ca793 100644
--- a/cpp/velox/memory/VeloxMemoryManager.h
+++ b/cpp/velox/memory/VeloxMemoryManager.h
@@ -27,10 +27,7 @@ namespace gluten {
class VeloxMemoryManager final : public MemoryManager {
public:
- VeloxMemoryManager(
- const std::string& name,
- std::shared_ptr<MemoryAllocator> allocator,
- std::unique_ptr<AllocationListener> listener);
+ VeloxMemoryManager(std::unique_ptr<AllocationListener> listener);
~VeloxMemoryManager() override;
VeloxMemoryManager(const VeloxMemoryManager&) = delete;
@@ -72,8 +69,6 @@ class VeloxMemoryManager final : public MemoryManager {
private:
bool tryDestructSafe();
- std::string name_;
-
#ifdef GLUTEN_ENABLE_HBM
std::unique_ptr<VeloxMemoryAllocator> wrappedAlloc_;
#endif
@@ -92,8 +87,7 @@ class VeloxMemoryManager final : public MemoryManager {
/// Not tracked by Spark and should only used in test or validation.
inline std::shared_ptr<gluten::VeloxMemoryManager> getDefaultMemoryManager() {
- static auto memoryManager = std::make_shared<gluten::VeloxMemoryManager>(
- "test", gluten::defaultMemoryAllocator(), gluten::AllocationListener::noop());
+ static auto memoryManager = std::make_shared<gluten::VeloxMemoryManager>(gluten::AllocationListener::noop());
return memoryManager;
}
diff --git a/cpp/velox/tests/MemoryManagerTest.cc b/cpp/velox/tests/MemoryManagerTest.cc
index 400beafcc..52f2fa8b6 100644
--- a/cpp/velox/tests/MemoryManagerTest.cc
+++ b/cpp/velox/tests/MemoryManagerTest.cc
@@ -54,7 +54,7 @@ class MemoryManagerTest : public ::testing::Test {
}
void SetUp() override {
- vmm_ = std::make_unique<VeloxMemoryManager>("test", stdAllocator_, std::make_unique<MockAllocationListener>());
+ vmm_ = std::make_unique<VeloxMemoryManager>(std::make_unique<MockAllocationListener>());
listener_ = vmm_->getListener();
allocator_ = vmm_->allocator();
}
diff --git a/cpp/velox/tests/RuntimeTest.cc b/cpp/velox/tests/RuntimeTest.cc
index 377d76054..563539d7d 100644
--- a/cpp/velox/tests/RuntimeTest.cc
+++ b/cpp/velox/tests/RuntimeTest.cc
@@ -18,19 +18,36 @@
#include "compute/VeloxRuntime.h"
#include <gtest/gtest.h>
+#include "compute/VeloxBackend.h"
namespace gluten {
+class DummyMemoryManager final : public MemoryManager {
+ public:
+ arrow::MemoryPool* getArrowMemoryPool() override {
+ throw GlutenException("Not yet implemented");
+ }
+ const MemoryUsageStats collectMemoryUsageStats() const override {
+ throw GlutenException("Not yet implemented");
+ }
+ const int64_t shrink(int64_t size) override {
+ throw GlutenException("Not yet implemented");
+ }
+ void hold() override {
+ throw GlutenException("Not yet implemented");
+ }
+};
+
class DummyRuntime final : public Runtime {
public:
- DummyRuntime(const std::unordered_map<std::string, std::string>& conf) : Runtime(conf) {}
+ DummyRuntime(std::unique_ptr<AllocationListener> listener, const std::unordered_map<std::string, std::string>& conf)
+ : Runtime(std::make_shared<DummyMemoryManager>(), conf) {}
void parsePlan(const uint8_t* data, int32_t size, std::optional<std::string> dumpFile) override {}
void parseSplitInfo(const uint8_t* data, int32_t size, std::optional<std::string> dumpFile) override {}
std::shared_ptr<ResultIterator> createResultIterator(
- MemoryManager* memoryManager,
const std::string& spillDir,
const std::vector<std::shared_ptr<ResultIterator>>& inputs,
const std::unordered_map<std::string, std::string>& sessionConf) override {
@@ -38,54 +55,41 @@ class DummyRuntime final : public Runtime {
auto iter = std::make_shared<ResultIterator>(std::move(resIter));
return iter;
}
- MemoryManager* createMemoryManager(
- const std::string& name,
- std::shared_ptr<MemoryAllocator> ptr,
- std::unique_ptr<AllocationListener> uniquePtr) override {
+ MemoryManager* memoryManager() override {
throw GlutenException("Not yet implemented");
}
std::shared_ptr<ColumnarBatch> createOrGetEmptySchemaBatch(int32_t numRows) override {
throw GlutenException("Not yet implemented");
}
- std::shared_ptr<ColumnarToRowConverter> createColumnar2RowConverter(MemoryManager* memoryManager) override {
+ std::shared_ptr<ColumnarToRowConverter> createColumnar2RowConverter() override {
throw GlutenException("Not yet implemented");
}
- std::shared_ptr<RowToColumnarConverter> createRow2ColumnarConverter(
- MemoryManager* memoryManager,
- struct ArrowSchema* cSchema) override {
+ std::shared_ptr<RowToColumnarConverter> createRow2ColumnarConverter(struct ArrowSchema* cSchema) override {
throw GlutenException("Not yet implemented");
}
std::shared_ptr<ShuffleWriter> createShuffleWriter(
int numPartitions,
std::unique_ptr<PartitionWriter> partitionWriter,
- ShuffleWriterOptions,
- MemoryManager* memoryManager) override {
+ ShuffleWriterOptions) override {
throw GlutenException("Not yet implemented");
}
Metrics* getMetrics(ColumnarBatchIterator* rawIter, int64_t exportNanos) override {
static Metrics m(1);
return &m;
}
- std::shared_ptr<Datasource> createDatasource(
- const std::string& filePath,
- MemoryManager* memoryManager,
- std::shared_ptr<arrow::Schema> schema) override {
+ std::shared_ptr<Datasource> createDatasource(const std::string& filePath, std::shared_ptr<arrow::Schema> schema)
+ override {
throw GlutenException("Not yet implemented");
}
std::shared_ptr<ShuffleReader> createShuffleReader(
std::shared_ptr<arrow::Schema> schema,
- ShuffleReaderOptions options,
- arrow::MemoryPool* pool,
- MemoryManager* memoryManager) override {
+ ShuffleReaderOptions options) override {
throw GlutenException("Not yet implemented");
}
- std::unique_ptr<ColumnarBatchSerializer> createColumnarBatchSerializer(
- MemoryManager* memoryManager,
- arrow::MemoryPool* arrowPool,
- struct ArrowSchema* cSchema) override {
+ std::unique_ptr<ColumnarBatchSerializer> createColumnarBatchSerializer(struct ArrowSchema* cSchema) override {
throw GlutenException("Not yet implemented");
}
- std::shared_ptr<ColumnarBatch> select(MemoryManager*, std::shared_ptr<ColumnarBatch>, std::vector<int32_t>) override {
+ std::shared_ptr<ColumnarBatch> select(std::shared_ptr<ColumnarBatch>, std::vector<int32_t>) override {
throw GlutenException("Not yet implemented");
}
std::string planString(bool details, const std::unordered_map<std::string, std::string>& sessionConf) override {
@@ -100,8 +104,6 @@ class DummyRuntime final : public Runtime {
}
private:
- ResourceMap<std::shared_ptr<ResultIterator>> resultIteratorHolder_;
-
class DummyResultIterator : public ColumnarBatchIterator {
public:
std::shared_ptr<ColumnarBatch> next() override {
@@ -118,20 +120,30 @@ class DummyRuntime final : public Runtime {
};
};
-static Runtime* dummyRuntimeFactory(const std::unordered_map<std::string, std::string> conf) {
- return new DummyRuntime(conf);
+static Runtime* dummyRuntimeFactory(
+ std::unique_ptr<AllocationListener> listener,
+ const std::unordered_map<std::string, std::string> conf) {
+ return new DummyRuntime(std::move(listener), conf);
}
TEST(TestRuntime, CreateRuntime) {
Runtime::registerFactory("DUMMY", dummyRuntimeFactory);
- auto runtime = Runtime::create("DUMMY");
+ auto runtime = Runtime::create("DUMMY", AllocationListener::noop());
ASSERT_EQ(typeid(*runtime), typeid(DummyRuntime));
Runtime::release(runtime);
}
+TEST(TestRuntime, CreateVeloxRuntime) {
+ VeloxBackend::create({});
+ auto runtime = Runtime::create(kVeloxRuntimeKind, AllocationListener::noop());
+ ASSERT_EQ(typeid(*runtime), typeid(VeloxRuntime));
+ Runtime::release(runtime);
+}
+
TEST(TestRuntime, GetResultIterator) {
- auto runtime = std::make_shared<DummyRuntime>(std::unordered_map<std::string, std::string>());
- auto iter = runtime->createResultIterator(nullptr, "/tmp/test-spill", {}, {});
+ auto runtime =
+ std::make_shared<DummyRuntime>(AllocationListener::noop(), std::unordered_map<std::string, std::string>());
+ auto iter = runtime->createResultIterator("/tmp/test-spill", {}, {});
ASSERT_TRUE(iter->hasNext());
auto next = iter->next();
ASSERT_NE(next, nullptr);
diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh
index 97b3db554..b812b6b52 100755
--- a/ep/build-velox/src/build_velox.sh
+++ b/ep/build-velox/src/build_velox.sh
@@ -302,4 +302,4 @@ check_commit
compile
echo "Successfully built Velox from Source."
-echo $TARGET_BUILD_SUMMARY >"${VELOX_HOME}/velox-build.cache"
+echo $TARGET_BUILD_SUMMARY > "${VELOX_HOME}/velox-build.cache"
diff --git a/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala b/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala
index 524a3ee2e..a7836e4a1 100644
--- a/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala
+++ b/gluten-celeborn/clickhouse/src/main/scala/org/apache/spark/shuffle/CHCelebornHashBasedColumnarShuffleWriter.scala
@@ -19,9 +19,7 @@ package org.apache.spark.shuffle
import org.apache.gluten.GlutenConfig
import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings
import org.apache.gluten.memory.alloc.CHNativeMemoryAllocators
-import org.apache.gluten.memory.memtarget.MemoryTarget
-import org.apache.gluten.memory.memtarget.Spiller
-import org.apache.gluten.memory.memtarget.Spillers
+import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers}
import org.apache.gluten.vectorized._
import org.apache.spark._
@@ -33,7 +31,6 @@ import org.apache.celeborn.client.ShuffleClient
import org.apache.celeborn.common.CelebornConf
import java.io.IOException
-import java.util
import java.util.Locale
class CHCelebornHashBasedColumnarShuffleWriter[K, V](
@@ -85,7 +82,10 @@ class CHCelebornHashBasedColumnarShuffleWriter[K, V](
CHNativeMemoryAllocators.createSpillable(
"CelebornShuffleWriter",
new Spiller() {
- override def spill(self: MemoryTarget, size: Long): Long = {
+ override def spill(self: MemoryTarget, phase: Spiller.Phase, size: Long): Long = {
+ if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) {
+ return 0L
+ }
if (nativeShuffleWriter == -1L) {
throw new IllegalStateException(
"Fatal: spill() called before a celeborn shuffle writer " +
@@ -98,8 +98,6 @@ class CHCelebornHashBasedColumnarShuffleWriter[K, V](
logInfo(s"Gluten shuffle writer: Spilled $spilled / $size bytes of data")
spilled
}
-
- override def applicablePhases(): util.Set[Spiller.Phase] = Spillers.PHASE_SET_SPILL_ONLY
}
)
}
diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
index 699626db1..1f125a164 100644
--- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
+++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornColumnarBatchSerializer.scala
@@ -19,7 +19,6 @@ package org.apache.spark.shuffle
import org.apache.gluten.GlutenConfig
import org.apache.gluten.exec.Runtimes
import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.ArrowAbiUtil
import org.apache.gluten.vectorized._
@@ -65,7 +64,7 @@ private class CelebornColumnarBatchSerializerInstance(
extends SerializerInstance
with Logging {
- private val nmm = NativeMemoryManagers.contextInstance("ShuffleReader")
+ private val runtime = Runtimes.contextInstance("CelebornShuffleReader")
private val shuffleReaderHandle = {
val allocator: BufferAllocator = ArrowBufferAllocators
@@ -86,12 +85,11 @@ private class CelebornColumnarBatchSerializerInstance(
GlutenConfig.getConf.columnarShuffleCodecBackend.orNull
val shuffleWriterType =
conf.get("spark.celeborn.client.spark.shuffle.writer", "hash").toLowerCase(Locale.ROOT)
- val jniWrapper = ShuffleReaderJniWrapper.create()
+ val jniWrapper = ShuffleReaderJniWrapper.create(runtime)
val batchSize = GlutenConfig.getConf.maxBatchSize
val handle = jniWrapper
.make(
cSchema.memoryAddress(),
- nmm.getNativeInstanceHandle,
compressionCodec,
compressionCodecBackend,
batchSize,
@@ -119,11 +117,10 @@ private class CelebornColumnarBatchSerializerInstance(
with TaskResource {
private val byteIn: JniByteInputStream = JniByteInputStreams.create(in)
private val wrappedOut: GeneralOutIterator = new ColumnarBatchOutIterator(
- Runtimes.contextInstance(),
+ runtime,
ShuffleReaderJniWrapper
- .create()
- .readStream(shuffleReaderHandle, byteIn),
- nmm)
+ .create(runtime)
+ .readStream(shuffleReaderHandle, byteIn))
private var cb: ColumnarBatch = _
diff --git a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala
index 37ea11a73..b8e6513cf 100644
--- a/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala
+++ b/gluten-celeborn/velox/src/main/scala/org/apache/spark/shuffle/VeloxCelebornHashBasedColumnarShuffleWriter.scala
@@ -18,10 +18,8 @@ package org.apache.spark.shuffle
import org.apache.gluten.GlutenConfig
import org.apache.gluten.columnarbatch.ColumnarBatches
-import org.apache.gluten.memory.memtarget.MemoryTarget
-import org.apache.gluten.memory.memtarget.Spiller
-import org.apache.gluten.memory.memtarget.Spillers
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
+import org.apache.gluten.exec.Runtimes
+import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers}
import org.apache.gluten.vectorized._
import org.apache.spark._
@@ -35,7 +33,6 @@ import org.apache.celeborn.client.ShuffleClient
import org.apache.celeborn.common.CelebornConf
import java.io.IOException
-import java.util
class VeloxCelebornHashBasedColumnarShuffleWriter[K, V](
shuffleId: Int,
@@ -52,7 +49,9 @@ class VeloxCelebornHashBasedColumnarShuffleWriter[K, V](
client,
writeMetrics) {
- private val jniWrapper = ShuffleWriterJniWrapper.create()
+ private val runtime = Runtimes.contextInstance("CelebornShuffleWriter")
+
+ private val jniWrapper = ShuffleWriterJniWrapper.create(runtime)
private var splitResult: SplitResult = _
@@ -105,31 +104,6 @@ class VeloxCelebornHashBasedColumnarShuffleWriter[K, V](
clientPushBufferMaxSize,
clientPushSortMemoryThreshold,
celebornPartitionPusher,
- NativeMemoryManagers
- .create(
- "CelebornShuffleWriter",
- new Spiller() {
- override def spill(self: MemoryTarget, size: Long): Long = {
- if (nativeShuffleWriter == -1L) {
- throw new IllegalStateException(
- "Fatal: spill() called before a celeborn shuffle writer " +
- "is created. This behavior should be" +
- "optimized by moving memory " +
- "allocations from make() to split()")
- }
- logInfo(s"Gluten shuffle writer: Trying to push $size bytes of data")
- // fixme pass true when being called by self
- val pushed =
- jniWrapper.nativeEvict(nativeShuffleWriter, size, false)
- logInfo(s"Gluten shuffle writer: Pushed $pushed / $size bytes of data")
- pushed
- }
-
- override def applicablePhases(): util.Set[Spiller.Phase] =
- Spillers.PHASE_SET_SPILL_ONLY
- }
- )
- .getNativeInstanceHandle,
handle,
context.taskAttemptId(),
GlutenShuffleUtils.getStartPartitionId(dep.nativePartitioning, context.partitionId),
@@ -137,6 +111,26 @@ class VeloxCelebornHashBasedColumnarShuffleWriter[K, V](
shuffleWriterType,
GlutenConfig.getConf.columnarShuffleReallocThreshold
)
+ runtime.addSpiller(new Spiller() {
+ override def spill(self: MemoryTarget, phase: Spiller.Phase, size: Long): Long = {
+ if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) {
+ return 0L
+ }
+ if (nativeShuffleWriter == -1L) {
+ throw new IllegalStateException(
+ "Fatal: spill() called before a celeborn shuffle writer " +
+ "is created. This behavior should be" +
+ "optimized by moving memory " +
+ "allocations from make() to split()")
+ }
+ logInfo(s"Gluten shuffle writer: Trying to push $size bytes of data")
+ // fixme pass true when being called by self
+ val pushed =
+ jniWrapper.nativeEvict(nativeShuffleWriter, size, false)
+ logInfo(s"Gluten shuffle writer: Pushed $pushed / $size bytes of data")
+ pushed
+ }
+ })
}
val startTime = System.nanoTime()
jniWrapper.write(nativeShuffleWriter, cb.numRows, handle, availableOffHeapPerTask())
diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
index c3ece7433..75e3db2e7 100644
--- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
+++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
@@ -22,7 +22,6 @@ import org.apache.gluten.memory.memtarget.spark.TreeMemoryConsumers;
import org.apache.spark.memory.TaskMemoryManager;
-import java.util.List;
import java.util.Map;
public final class MemoryTargets {
@@ -54,7 +53,7 @@ public final class MemoryTargets {
public static MemoryTarget newConsumer(
TaskMemoryManager tmm,
String name,
- List<Spiller> spillers,
+ Spiller spiller,
Map<String, MemoryUsageStatsBuilder> virtualChildren) {
final TreeMemoryConsumers.Factory factory;
if (GlutenConfig.getConf().memoryIsolation()) {
@@ -63,6 +62,6 @@ public final class MemoryTargets {
factory = TreeMemoryConsumers.shared();
}
- return factory.newConsumer(tmm, name, spillers, virtualChildren);
+ return factory.newConsumer(tmm, name, spiller, virtualChildren);
}
}
diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java
index 0e0457a7c..a0ec6dc6c 100644
--- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java
+++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spiller.java
@@ -16,14 +16,11 @@
*/
package org.apache.gluten.memory.memtarget;
-import java.util.Set;
-
public interface Spiller {
+ long spill(MemoryTarget self, Phase phase, long size);
- long spill(MemoryTarget self, long size);
-
- Set<Phase> applicablePhases();
-
+ // Order of the elements matters, since
+ // consumer should call spillers with in the defined order.
enum Phase {
SHRINK,
SPILL
diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java
index 6032b9494..4477e2956 100644
--- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java
+++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/Spillers.java
@@ -16,16 +16,21 @@
*/
package org.apache.gluten.memory.memtarget;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.*;
public final class Spillers {
private Spillers() {
// enclose factory ctor
}
+ public static final Spiller NOOP =
+ new Spiller() {
+ @Override
+ public long spill(MemoryTarget self, Phase phase, long size) {
+ return 0;
+ }
+ };
+
public static final Set<Spiller.Phase> PHASE_SET_ALL =
Collections.unmodifiableSet(
new HashSet<>(Arrays.asList(Spiller.Phase.SHRINK, Spiller.Phase.SPILL)));
@@ -40,6 +45,10 @@ public final class Spillers {
return new WithMinSpillSize(spiller, minSize);
}
+ public static AppendableSpillerList appendable() {
+ return new AppendableSpillerList();
+ }
+
// Minimum spill target size should be larger than spark.gluten.memory.reservationBlockSize,
// since any release action within size smaller than the block size may not have chance to
// report back to the Java-side reservation listener.
@@ -53,13 +62,27 @@ public final class Spillers {
}
@Override
- public long spill(MemoryTarget self, long size) {
- return delegated.spill(self, Math.max(size, minSize));
+ public long spill(MemoryTarget self, Spiller.Phase phase, long size) {
+ return delegated.spill(self, phase, Math.max(size, minSize));
+ }
+ }
+
+ public static class AppendableSpillerList implements Spiller {
+ private final List<Spiller> spillers = new LinkedList<>();
+
+ private AppendableSpillerList() {}
+
+ public void append(Spiller spiller) {
+ spillers.add(spiller);
}
@Override
- public Set<Phase> applicablePhases() {
- return delegated.applicablePhases();
+ public long spill(MemoryTarget self, Phase phase, final long size) {
+ long remainingBytes = size;
+ for (Spiller spiller : spillers) {
+ remainingBytes -= spiller.spill(self, phase, remainingBytes);
+ }
+ return size - remainingBytes;
}
}
}
diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java
index 37d44fd7a..9543bb5ed 100644
--- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java
+++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTarget.java
@@ -19,7 +19,6 @@ package org.apache.gluten.memory.memtarget;
import org.apache.gluten.memory.MemoryUsageStatsBuilder;
import org.apache.gluten.memory.memtarget.spark.TreeMemoryConsumer;
-import java.util.List;
import java.util.Map;
/** An abstract for both {@link TreeMemoryConsumer} and it's non-consumer children nodes. */
@@ -29,12 +28,12 @@ public interface TreeMemoryTarget extends MemoryTarget, KnownNameAndStats {
TreeMemoryTarget newChild(
String name,
long capacity,
- List<Spiller> spillers,
+ Spiller spiller,
Map<String, MemoryUsageStatsBuilder> virtualChildren);
Map<String, TreeMemoryTarget> children();
TreeMemoryTarget parent();
- List<Spiller> getNodeSpillers();
+ Spiller getNodeSpiller();
}
diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java
index a8dc39ce3..24d9fc0e2 100644
--- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java
+++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/TreeMemoryTargets.java
@@ -23,19 +23,14 @@ import org.apache.gluten.proto.MemoryUsageStats;
import com.google.common.base.Preconditions;
import org.apache.spark.util.Utils;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.Queue;
-import java.util.function.Predicate;
import java.util.stream.Collectors;
public class TreeMemoryTargets {
- public static final List<Spiller.Phase> SPILL_PHASES =
- Arrays.asList(Spiller.Phase.SHRINK, Spiller.Phase.SPILL);
private TreeMemoryTargets() {
// enclose factory ctor
@@ -45,26 +40,24 @@ public class TreeMemoryTargets {
TreeMemoryTarget parent,
String name,
long capacity,
- List<Spiller> spillers,
+ Spiller spiller,
Map<String, MemoryUsageStatsBuilder> virtualChildren) {
- return new Node(parent, name, capacity, spillers, virtualChildren);
+ return new Node(parent, name, capacity, spiller, virtualChildren);
}
public static long spillTree(TreeMemoryTarget node, final long bytes) {
long remainingBytes = bytes;
- for (Spiller.Phase phase : SPILL_PHASES) {
+ for (Spiller.Phase phase : Spiller.Phase.values()) {
// First shrink, then if no good, spill.
if (remainingBytes <= 0) {
break;
}
- remainingBytes -=
- spillTree(node, remainingBytes, spiller -> spiller.applicablePhases().contains(phase));
+ remainingBytes -= spillTree(node, phase, remainingBytes);
}
return bytes - remainingBytes;
}
- private static long spillTree(
- TreeMemoryTarget node, final long bytes, Predicate<Spiller> spillerFilter) {
+ private static long spillTree(TreeMemoryTarget node, Spiller.Phase phase, final long bytes) {
// sort children by used bytes, descending
Queue<TreeMemoryTarget> q =
new PriorityQueue<>(
@@ -83,13 +76,9 @@ public class TreeMemoryTargets {
if (remainingBytes > 0) {
// if still doesn't fit, spill self
- final List<Spiller> applicableSpillers =
- node.getNodeSpillers().stream().filter(spillerFilter).collect(Collectors.toList());
- for (int i = 0; i < applicableSpillers.size() && remainingBytes > 0; i++) {
- final Spiller spiller = applicableSpillers.get(i);
- long spilled = spiller.spill(node, remainingBytes);
- remainingBytes -= spilled;
- }
+ final Spiller spiller = node.getNodeSpiller();
+ long spilled = spiller.spill(node, phase, remainingBytes);
+ remainingBytes -= spilled;
}
return bytes - remainingBytes;
@@ -101,7 +90,7 @@ public class TreeMemoryTargets {
private final TreeMemoryTarget parent;
private final String name;
private final long capacity;
- private final List<Spiller> spillers;
+ private final Spiller spiller;
private final Map<String, MemoryUsageStatsBuilder> virtualChildren;
private final SimpleMemoryUsageRecorder selfRecorder = new SimpleMemoryUsageRecorder();
@@ -109,7 +98,7 @@ public class TreeMemoryTargets {
TreeMemoryTarget parent,
String name,
long capacity,
- List<Spiller> spillers,
+ Spiller spiller,
Map<String, MemoryUsageStatsBuilder> virtualChildren) {
this.parent = parent;
this.capacity = capacity;
@@ -119,7 +108,7 @@ public class TreeMemoryTargets {
} else {
this.name = String.format("%s, %s", uniqueName, Utils.bytesToString(capacity));
}
- this.spillers = Collections.unmodifiableList(spillers);
+ this.spiller = spiller;
this.virtualChildren = virtualChildren;
}
@@ -140,8 +129,8 @@ public class TreeMemoryTargets {
}
@Override
- public List<Spiller> getNodeSpillers() {
- return spillers;
+ public Spiller getNodeSpiller() {
+ return spiller;
}
private boolean ensureFreeCapacity(long bytesNeeded) {
@@ -209,9 +198,9 @@ public class TreeMemoryTargets {
public TreeMemoryTarget newChild(
String name,
long capacity,
- List<Spiller> spillers,
+ Spiller spiller,
Map<String, MemoryUsageStatsBuilder> virtualChildren) {
- final Node child = new Node(this, name, capacity, spillers, virtualChildren);
+ final Node child = new Node(this, name, capacity, spiller, virtualChildren);
if (children.containsKey(child.name())) {
throw new IllegalArgumentException("Child already registered: " + child.name());
}
diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java
index 3028547c0..76aa63aeb 100644
--- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java
+++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/RegularMemoryConsumer.java
@@ -63,8 +63,16 @@ public class RegularMemoryConsumer extends MemoryConsumer
}
@Override
- public long spill(long size, MemoryConsumer trigger) {
- long spilledOut = spiller.spill(this, size);
+ public long spill(final long size, MemoryConsumer trigger) {
+ long remainingBytes = size;
+ for (Spiller.Phase phase : Spiller.Phase.values()) {
+ // First shrink, then if no good, spill.
+ if (remainingBytes <= 0) {
+ break;
+ }
+ remainingBytes -= spiller.spill(this, phase, size);
+ }
+ long spilledOut = size - remainingBytes;
if (TaskResources.inSparkTask()) {
TaskResources.getLocalTaskContext().taskMetrics().incMemoryBytesSpilled(spilledOut);
}
diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
index f86295b06..44c725798 100644
--- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
+++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
@@ -18,11 +18,7 @@ package org.apache.gluten.memory.memtarget.spark;
import org.apache.gluten.memory.MemoryUsageStatsBuilder;
import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
-import org.apache.gluten.memory.memtarget.MemoryTargetUtil;
-import org.apache.gluten.memory.memtarget.MemoryTargetVisitor;
-import org.apache.gluten.memory.memtarget.Spiller;
-import org.apache.gluten.memory.memtarget.TreeMemoryTarget;
-import org.apache.gluten.memory.memtarget.TreeMemoryTargets;
+import org.apache.gluten.memory.memtarget.*;
import org.apache.gluten.proto.MemoryUsageStats;
import com.google.common.base.Preconditions;
@@ -33,7 +29,6 @@ import org.apache.spark.memory.TaskMemoryManager;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@@ -124,10 +119,10 @@ public class TreeMemoryConsumer extends MemoryConsumer implements TreeMemoryTarg
public TreeMemoryTarget newChild(
String name,
long capacity,
- List<Spiller> spillers,
+ Spiller spiller,
Map<String, MemoryUsageStatsBuilder> virtualChildren) {
final TreeMemoryTarget child =
- TreeMemoryTargets.newChild(this, name, capacity, spillers, virtualChildren);
+ TreeMemoryTargets.newChild(this, name, capacity, spiller, virtualChildren);
if (children.containsKey(child.name())) {
throw new IllegalArgumentException("Child already registered: " + child.name());
}
@@ -147,9 +142,9 @@ public class TreeMemoryConsumer extends MemoryConsumer implements TreeMemoryTarg
}
@Override
- public List<Spiller> getNodeSpillers() {
+ public Spiller getNodeSpiller() {
// root doesn't spill
- return Collections.emptyList();
+ return Spillers.NOOP;
}
public TaskMemoryManager getTaskMemoryManager() {
diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
index 1da23d15e..7ab05bd3a 100644
--- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
+++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
@@ -19,13 +19,13 @@ package org.apache.gluten.memory.memtarget.spark;
import org.apache.gluten.GlutenConfig;
import org.apache.gluten.memory.MemoryUsageStatsBuilder;
import org.apache.gluten.memory.memtarget.Spiller;
+import org.apache.gluten.memory.memtarget.Spillers;
import org.apache.gluten.memory.memtarget.TreeMemoryTarget;
import org.apache.commons.collections.map.ReferenceMap;
import org.apache.spark.memory.TaskMemoryManager;
import java.util.Collections;
-import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -76,7 +76,7 @@ public final class TreeMemoryConsumers {
m -> {
TreeMemoryTarget tmc = new TreeMemoryConsumer((TaskMemoryManager) m);
return tmc.newChild(
- "root", perTaskCapacity, Collections.emptyList(), Collections.emptyMap());
+ "root", perTaskCapacity, Spillers.NOOP, Collections.emptyMap());
});
}
}
@@ -84,11 +84,11 @@ public final class TreeMemoryConsumers {
public TreeMemoryTarget newConsumer(
TaskMemoryManager tmm,
String name,
- List<Spiller> spillers,
+ Spiller spiller,
Map<String, MemoryUsageStatsBuilder> virtualChildren) {
final TreeMemoryTarget account = getSharedAccount(tmm);
return account.newChild(
- name, TreeMemoryConsumer.CAPACITY_UNLIMITED, spillers, virtualChildren);
+ name, TreeMemoryConsumer.CAPACITY_UNLIMITED, spiller, virtualChildren);
}
}
}
diff --git a/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
index 48ed08fb7..178310fd6 100644
--- a/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
+++ b/gluten-core/src/main/scala/org/apache/spark/memory/SparkMemoryUtil.scala
@@ -44,6 +44,10 @@ object SparkMemoryUtil {
consumersField.setAccessible(true)
taskIdField.setAccessible(true)
+ def bytesToString(size: Long): String = {
+ Utils.bytesToString(size)
+ }
+
// We assume storage memory can be fully transferred to execution memory so far
def getCurrentAvailableOffHeapMemory: Long = {
val mm = SparkEnv.get.memoryManager
diff --git a/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java b/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java
index e26765d33..db018ffe4 100644
--- a/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java
+++ b/gluten-core/src/test/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumerTest.java
@@ -17,6 +17,7 @@
package org.apache.gluten.memory.memtarget.spark;
import org.apache.gluten.GlutenConfig;
+import org.apache.gluten.memory.memtarget.Spillers;
import org.apache.gluten.memory.memtarget.TreeMemoryTarget;
import org.apache.spark.TaskContext;
@@ -43,7 +44,7 @@ public class TreeMemoryConsumerTest {
factory.newConsumer(
TaskContext.get().taskMemoryManager(),
"FOO",
- Collections.emptyList(),
+ Spillers.NOOP,
Collections.emptyMap());
Assert.assertEquals(20, consumer.borrow(20));
Assert.assertEquals(70, consumer.borrow(70));
@@ -65,7 +66,7 @@ public class TreeMemoryConsumerTest {
factory.newConsumer(
TaskContext.get().taskMemoryManager(),
"FOO",
- Collections.emptyList(),
+ Spillers.NOOP,
Collections.emptyMap());
Assert.assertEquals(20, consumer.borrow(20));
Assert.assertEquals(70, consumer.borrow(70));
@@ -87,7 +88,7 @@ public class TreeMemoryConsumerTest {
.newConsumer(
TaskContext.get().taskMemoryManager(),
"FOO",
- Collections.emptyList(),
+ Spillers.NOOP,
Collections.emptyMap());
Assert.assertEquals(110, shared.borrow(110));
final TreeMemoryTarget isolated =
@@ -95,7 +96,7 @@ public class TreeMemoryConsumerTest {
.newConsumer(
TaskContext.get().taskMemoryManager(),
"FOO",
- Collections.emptyList(),
+ Spillers.NOOP,
Collections.emptyMap());
Assert.assertEquals(100, isolated.borrow(110));
});
diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java
index 4f042b893..a834e13a4 100644
--- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java
+++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatchJniWrapper.java
@@ -18,7 +18,6 @@ package org.apache.gluten.columnarbatch;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.RuntimeAware;
-import org.apache.gluten.exec.Runtimes;
public class ColumnarBatchJniWrapper implements RuntimeAware {
private final Runtime runtime;
@@ -27,11 +26,7 @@ public class ColumnarBatchJniWrapper implements RuntimeAware {
this.runtime = runtime;
}
- public static ColumnarBatchJniWrapper create() {
- return new ColumnarBatchJniWrapper(Runtimes.contextInstance());
- }
-
- public static ColumnarBatchJniWrapper forRuntime(Runtime runtime) {
+ public static ColumnarBatchJniWrapper create(Runtime runtime) {
return new ColumnarBatchJniWrapper(runtime);
}
@@ -39,22 +34,21 @@ public class ColumnarBatchJniWrapper implements RuntimeAware {
public native long getForEmptySchema(int numRows);
- public native String getType(long batchHandle);
+ public native String getType(long batch);
- public native long numColumns(long batchHandle);
+ public native long numColumns(long batch);
- public native long numRows(long batchHandle);
+ public native long numRows(long batch);
- public native long numBytes(long batchHandle);
+ public native long numBytes(long batch);
public native long compose(long[] batches);
public native void exportToArrow(long batch, long cSchema, long cArray);
- public native long select(
- long nativeMemoryManagerHandle, // why a mm is needed here?
- long batch,
- int[] columnIndices);
+ public native long select(long batch, int[] columnIndices);
+
+ public native long obtainOwnership(long batch);
public native void close(long batch);
diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java
index e2cfa335d..fc3b56c1b 100644
--- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java
+++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/ColumnarBatches.java
@@ -19,13 +19,11 @@ package org.apache.gluten.columnarbatch;
import org.apache.gluten.exception.GlutenException;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.Runtimes;
-import org.apache.gluten.memory.nmm.NativeMemoryManager;
import org.apache.gluten.utils.ArrowAbiUtil;
import org.apache.gluten.utils.ArrowUtil;
import org.apache.gluten.utils.ImplicitClass;
import org.apache.gluten.vectorized.ArrowWritableColumnVector;
-import com.google.common.base.Preconditions;
import org.apache.arrow.c.ArrowArray;
import org.apache.arrow.c.ArrowSchema;
import org.apache.arrow.c.CDataDictionaryProvider;
@@ -38,11 +36,8 @@ import org.apache.spark.sql.vectorized.ColumnarBatch;
import java.lang.reflect.Field;
import java.util.Arrays;
-import java.util.HashSet;
import java.util.Iterator;
-import java.util.List;
import java.util.NoSuchElementException;
-import java.util.Set;
public class ColumnarBatches {
private static final Field FIELD_COLUMNS;
@@ -125,15 +120,14 @@ public class ColumnarBatches {
* This method will always return a velox based ColumnarBatch. This method will close the input
* column batch.
*/
- public static ColumnarBatch select(
- NativeMemoryManager nmm, ColumnarBatch batch, int[] columnIndices) {
+ public static ColumnarBatch select(ColumnarBatch batch, int[] columnIndices) {
+ final Runtime runtime = Runtimes.contextInstance("ColumnarBatches#select");
switch (identifyBatchType(batch)) {
case LIGHT:
final IndicatorVector iv = getIndicatorVector(batch);
long outputBatchHandle =
- ColumnarBatchJniWrapper.create()
- .select(nmm.getNativeInstanceHandle(), iv.handle(), columnIndices);
- return create(iv.runtime(), outputBatchHandle);
+ ColumnarBatchJniWrapper.create(runtime).select(iv.handle(), columnIndices);
+ return create(runtime, outputBatchHandle);
case HEAVY:
return new ColumnarBatch(
Arrays.stream(columnIndices).mapToObj(batch::column).toArray(ColumnVector[]::new),
@@ -181,7 +175,7 @@ public class ColumnarBatches {
ArrowArray cArray = ArrowArray.allocateNew(allocator);
ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator);
CDataDictionaryProvider provider = new CDataDictionaryProvider()) {
- ColumnarBatchJniWrapper.forRuntime(iv.runtime())
+ ColumnarBatchJniWrapper.create(Runtimes.contextInstance("ColumnarBatches#load"))
.exportToArrow(iv.handle(), cSchema.memoryAddress(), cArray.memoryAddress());
Data.exportSchema(
@@ -217,12 +211,12 @@ public class ColumnarBatches {
if (input.numCols() == 0) {
throw new IllegalArgumentException("batch with zero columns cannot be offloaded");
}
- final Runtime runtime = Runtimes.contextInstance();
+ final Runtime runtime = Runtimes.contextInstance("ColumnarBatches#offload");
try (ArrowArray cArray = ArrowArray.allocateNew(allocator);
ArrowSchema cSchema = ArrowSchema.allocateNew(allocator)) {
ArrowAbiUtil.exportFromSparkColumnarBatch(allocator, input, cSchema, cArray);
long handle =
- ColumnarBatchJniWrapper.forRuntime(runtime)
+ ColumnarBatchJniWrapper.create(runtime)
.createWithArrowArray(cSchema.memoryAddress(), cArray.memoryAddress());
ColumnarBatch output = ColumnarBatches.create(runtime, handle);
@@ -333,18 +327,12 @@ public class ColumnarBatches {
Arrays.stream(batches)
.map(ColumnarBatches::getIndicatorVector)
.toArray(IndicatorVector[]::new);
- // We assume all input batches should be managed by same Runtime.
- // FIXME: The check could be removed to adopt ownership-transfer semantic
- final Runtime[] ctxs =
- Arrays.stream(ivs).map(IndicatorVector::runtime).distinct().toArray(Runtime[]::new);
- Preconditions.checkState(
- ctxs.length == 1, "All input batches should be managed by same Runtime.");
final long[] handles = Arrays.stream(ivs).mapToLong(IndicatorVector::handle).toArray();
- return ColumnarBatchJniWrapper.forRuntime(ctxs[0]).compose(handles);
+ return ColumnarBatchJniWrapper.create(Runtimes.contextInstance("ColumnarBatches#compose"))
+ .compose(handles);
}
- public static ColumnarBatch create(Runtime runtime, long nativeHandle) {
- final IndicatorVector iv = new IndicatorVector(runtime, nativeHandle);
+ private static ColumnarBatch create(IndicatorVector iv) {
int numColumns = Math.toIntExact(iv.getNumColumns());
int numRows = Math.toIntExact(iv.getNumRows());
if (numColumns == 0) {
@@ -360,6 +348,10 @@ public class ColumnarBatches {
return new ColumnarBatch(columnVectors, numRows);
}
+ public static ColumnarBatch create(Runtime runtime, long nativeHandle) {
+ return create(new IndicatorVector(runtime, nativeHandle));
+ }
+
public static void retain(ColumnarBatch b) {
switch (identifyBatchType(b)) {
case LIGHT:
@@ -384,18 +376,4 @@ public class ColumnarBatches {
public static long getNativeHandle(ColumnarBatch batch) {
return getIndicatorVector(batch).handle();
}
-
- public static Runtime getRuntime(ColumnarBatch batch) {
- return getIndicatorVector(batch).runtime();
- }
-
- public static Runtime getRuntime(List<ColumnarBatch> batch) {
- final Set<Runtime> all = new HashSet<>();
- batch.forEach(b -> all.add(getRuntime(b)));
- if (all.size() != 1) {
- throw new IllegalArgumentException(
- "The input columnar batches has different associated runtimes");
- }
- return all.toArray(new Runtime[0])[0];
- }
}
diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java
index d15412bec..0ec5b78ce 100644
--- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java
+++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java
@@ -18,146 +18,33 @@ package org.apache.gluten.columnarbatch;
import org.apache.gluten.exec.Runtime;
-import org.apache.spark.sql.types.DataTypes;
-import org.apache.spark.sql.types.Decimal;
-import org.apache.spark.sql.vectorized.ColumnVector;
-import org.apache.spark.sql.vectorized.ColumnarArray;
-import org.apache.spark.sql.vectorized.ColumnarMap;
-import org.apache.spark.unsafe.types.UTF8String;
-
import java.util.concurrent.atomic.AtomicLong;
-public class IndicatorVector extends ColumnVector {
- private final Runtime runtime;
- private final long handle;
+public class IndicatorVector extends IndicatorVectorBase {
private final AtomicLong refCnt = new AtomicLong(1L);
protected IndicatorVector(Runtime runtime, long handle) {
- super(DataTypes.NullType);
- this.runtime = runtime;
- this.handle = handle;
- }
-
- public Runtime runtime() {
- return runtime;
- }
-
- public String getType() {
- return ColumnarBatchJniWrapper.forRuntime(runtime).getType(handle);
- }
-
- public long getNumColumns() {
- return ColumnarBatchJniWrapper.forRuntime(runtime).numColumns(handle);
- }
-
- public long getNumRows() {
- return ColumnarBatchJniWrapper.forRuntime(runtime).numRows(handle);
+ super(runtime, handle);
}
- public long refCnt() {
+ @Override
+ long refCnt() {
return refCnt.get();
}
- public void retain() {
+ @Override
+ void retain() {
refCnt.getAndIncrement();
}
@Override
- public void close() {
+ void release() {
if (refCnt.get() == 0) {
// TODO use stronger restriction (IllegalStateException probably)
return;
}
if (refCnt.decrementAndGet() == 0) {
- ColumnarBatchJniWrapper.forRuntime(runtime).close(handle);
+ jniWrapper.close(handle);
}
}
-
- public boolean isClosed() {
- return refCnt.get() == 0;
- }
-
- @Override
- public boolean hasNull() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int numNulls() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean isNullAt(int rowId) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean getBoolean(int rowId) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public byte getByte(int rowId) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public short getShort(int rowId) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int getInt(int rowId) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long getLong(int rowId) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public float getFloat(int rowId) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public double getDouble(int rowId) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ColumnarArray getArray(int rowId) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ColumnarMap getMap(int ordinal) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Decimal getDecimal(int rowId, int precision, int scale) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public UTF8String getUTF8String(int rowId) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public byte[] getBinary(int rowId) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ColumnVector getChild(int ordinal) {
- throw new UnsupportedOperationException();
- }
-
- public long handle() {
- return handle;
- }
}
diff --git a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java
similarity index 69%
copy from gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java
copy to gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java
index d15412bec..fa695127a 100644
--- a/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVector.java
+++ b/gluten-data/src/main/java/org/apache/gluten/columnarbatch/IndicatorVectorBase.java
@@ -25,56 +25,59 @@ import org.apache.spark.sql.vectorized.ColumnarArray;
import org.apache.spark.sql.vectorized.ColumnarMap;
import org.apache.spark.unsafe.types.UTF8String;
-import java.util.concurrent.atomic.AtomicLong;
-
-public class IndicatorVector extends ColumnVector {
+public abstract class IndicatorVectorBase extends ColumnVector {
private final Runtime runtime;
- private final long handle;
- private final AtomicLong refCnt = new AtomicLong(1L);
+ protected final long handle;
+ protected final ColumnarBatchJniWrapper jniWrapper;
- protected IndicatorVector(Runtime runtime, long handle) {
+ protected IndicatorVectorBase(Runtime runtime, long handle) {
super(DataTypes.NullType);
this.runtime = runtime;
- this.handle = handle;
- }
-
- public Runtime runtime() {
- return runtime;
+ this.jniWrapper = ColumnarBatchJniWrapper.create(runtime);
+ this.handle = takeOwnership(handle);
+ }
+
+ private long takeOwnership(long handle) {
+ // Note: Underlying memory of returned batch still holds
+ // reference to the original memory manager. As
+ // a result, once its original resident runtime / mm is
+ // released, data may become invalid. Currently, it's
+ // the caller's responsibility to make sure the original
+ // runtime / mm keep alive even this function
+ // was called.
+ //
+ // Additionally, as in Gluten we have principle that runtime
+ // mm that were created earlier will be released
+ // later, this FILO practice is what helps the runtime that
+ // took ownership be able to access the data constantly
+ // because the original runtime will live longer than
+ // itself.
+ long newHandle = jniWrapper.obtainOwnership(handle);
+ jniWrapper.close(handle);
+ return newHandle;
}
public String getType() {
- return ColumnarBatchJniWrapper.forRuntime(runtime).getType(handle);
+ return jniWrapper.getType(handle);
}
public long getNumColumns() {
- return ColumnarBatchJniWrapper.forRuntime(runtime).numColumns(handle);
+ return jniWrapper.numColumns(handle);
}
public long getNumRows() {
- return ColumnarBatchJniWrapper.forRuntime(runtime).numRows(handle);
+ return jniWrapper.numRows(handle);
}
- public long refCnt() {
- return refCnt.get();
- }
+ abstract long refCnt();
- public void retain() {
- refCnt.getAndIncrement();
- }
+ abstract void retain();
+
+ abstract void release();
@Override
public void close() {
- if (refCnt.get() == 0) {
- // TODO use stronger restriction (IllegalStateException probably)
- return;
- }
- if (refCnt.decrementAndGet() == 0) {
- ColumnarBatchJniWrapper.forRuntime(runtime).close(handle);
- }
- }
-
- public boolean isClosed() {
- return refCnt.get() == 0;
+ release();
}
@Override
diff --git a/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java
index 46d12c1de..5a34196c4 100644
--- a/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java
+++ b/gluten-data/src/main/java/org/apache/gluten/datasource/DatasourceJniWrapper.java
@@ -18,7 +18,6 @@ package org.apache.gluten.datasource;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.RuntimeAware;
-import org.apache.gluten.exec.Runtimes;
import org.apache.gluten.init.JniUtils;
import org.apache.gluten.vectorized.ColumnarBatchInIterator;
@@ -35,8 +34,8 @@ public class DatasourceJniWrapper implements RuntimeAware {
this.runtime = runtime;
}
- public static DatasourceJniWrapper create() {
- return new DatasourceJniWrapper(Runtimes.contextInstance());
+ public static DatasourceJniWrapper create(Runtime runtime) {
+ return new DatasourceJniWrapper(runtime);
}
@Override
@@ -44,14 +43,11 @@ public class DatasourceJniWrapper implements RuntimeAware {
return runtime.getHandle();
}
- public long nativeInitDatasource(
- String filePath, long cSchema, long memoryManagerHandle, Map<String, String> options) {
- return nativeInitDatasource(
- filePath, cSchema, memoryManagerHandle, JniUtils.toNativeConf(options));
+ public long nativeInitDatasource(String filePath, long cSchema, Map<String, String> options) {
+ return nativeInitDatasource(filePath, cSchema, JniUtils.toNativeConf(options));
}
- public native long nativeInitDatasource(
- String filePath, long cSchema, long memoryManagerHandle, byte[] options);
+ public native long nativeInitDatasource(String filePath, long cSchema, byte[] options);
public native void inspectSchema(long dsHandle, long cSchemaAddress);
@@ -60,5 +56,5 @@ public class DatasourceJniWrapper implements RuntimeAware {
public native void write(long dsHandle, ColumnarBatchInIterator iterator);
public native BlockStripes splitBlockByPartitionAndBucket(
- long blockAddress, int[] partitionColIndice, boolean hasBucket, long memoryManagerId);
+ long blockAddress, int[] partitionColIndice, boolean hasBucket);
}
diff --git a/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeJniWrapper.java
index 14ec87dcf..d2a18e9b4 100644
--- a/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeJniWrapper.java
+++ b/gluten-data/src/main/java/org/apache/gluten/exec/RuntimeJniWrapper.java
@@ -16,11 +16,21 @@
*/
package org.apache.gluten.exec;
+import org.apache.gluten.memory.listener.ReservationListener;
+
public class RuntimeJniWrapper {
private RuntimeJniWrapper() {}
- public static native long createRuntime(String backendType, byte[] sessionConf);
+ public static native long createRuntime(
+ String backendType, ReservationListener listener, byte[] sessionConf);
+
+ // Memory management.
+ public static native byte[] collectMemoryUsage(long handle);
+
+ public static native long shrinkMemory(long handle, long size);
+
+ public static native void holdMemory(long handle);
public static native void releaseRuntime(long handle);
}
diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocator.java b/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocator.java
deleted file mode 100644
index 6860d68e3..000000000
--- a/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocator.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.memory.alloc;
-
-/**
- * This along with {@link NativeMemoryAllocators}, as built-in toolkit for managing native memory
- * allocations.
- */
-public class NativeMemoryAllocator {
- enum Type {
- DEFAULT,
- }
-
- private final long nativeInstanceId;
-
- public NativeMemoryAllocator(long nativeInstanceId) {
- this.nativeInstanceId = nativeInstanceId;
- }
-
- public static NativeMemoryAllocator create(Type type) {
- return new NativeMemoryAllocator(getAllocator(type.name()));
- }
-
- public long getNativeInstanceId() {
- return this.nativeInstanceId;
- }
-
- public void close() {
- releaseAllocator(this.nativeInstanceId);
- }
-
- private static native long getAllocator(String typeName);
-
- private static native void releaseAllocator(long allocatorId);
-}
diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocators.java b/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocators.java
deleted file mode 100644
index e60766ac1..000000000
--- a/gluten-data/src/main/java/org/apache/gluten/memory/alloc/NativeMemoryAllocators.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.memory.alloc;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Built-in toolkit for managing native memory allocations. To use the facility, one should import
- * Gluten's C++ library then create the c++ instance using following example code:
- *
- * <p>```c++ auto* allocator = reinterpret_cast<gluten::memory::MemoryAllocator*>(allocator_id); ```
- *
- * <p>The ID "allocator_id" can be retrieved from Java API {@link
- * NativeMemoryAllocator#getNativeInstanceId()}.
- *
- * <p>FIXME: to export the native APIs in a standard way
- */
-public final class NativeMemoryAllocators {
- private static final Map<NativeMemoryAllocator.Type, NativeMemoryAllocators> INSTANCES =
- new ConcurrentHashMap<>();
-
- private final NativeMemoryAllocator allocator;
-
- private NativeMemoryAllocators(NativeMemoryAllocator.Type type) {
- allocator = NativeMemoryAllocator.create(type);
- }
-
- public static NativeMemoryAllocators getDefault() {
- return forType(NativeMemoryAllocator.Type.DEFAULT);
- }
-
- private static NativeMemoryAllocators forType(NativeMemoryAllocator.Type type) {
- return INSTANCES.computeIfAbsent(type, NativeMemoryAllocators::new);
- }
-
- public NativeMemoryAllocator get() {
- return allocator;
- }
-}
diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java
index 51f49da70..f51852ab8 100644
--- a/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java
+++ b/gluten-data/src/main/java/org/apache/gluten/memory/arrow/alloc/ArrowBufferAllocators.java
@@ -17,6 +17,7 @@
package org.apache.gluten.memory.arrow.alloc;
import org.apache.gluten.memory.memtarget.MemoryTargets;
+import org.apache.gluten.memory.memtarget.Spillers;
import org.apache.arrow.memory.AllocationListener;
import org.apache.arrow.memory.BufferAllocator;
@@ -62,10 +63,7 @@ public class ArrowBufferAllocators {
MemoryTargets.throwOnOom(
MemoryTargets.dynamicOffHeapSizingIfEnabled(
MemoryTargets.newConsumer(
- tmm,
- "ArrowContextInstance",
- Collections.emptyList(),
- Collections.emptyMap()))),
+ tmm, "ArrowContextInstance", Spillers.NOOP, Collections.emptyMap()))),
TaskResources.getSharedUsage());
}
diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/ManagedReservationListener.java b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ManagedReservationListener.java
similarity index 98%
rename from gluten-data/src/main/java/org/apache/gluten/memory/nmm/ManagedReservationListener.java
rename to gluten-data/src/main/java/org/apache/gluten/memory/listener/ManagedReservationListener.java
index 88e793320..b7d6ecd67 100644
--- a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/ManagedReservationListener.java
+++ b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ManagedReservationListener.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.memory.nmm;
+package org.apache.gluten.memory.listener;
import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
import org.apache.gluten.memory.memtarget.MemoryTarget;
diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/ReservationListener.java b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListener.java
similarity index 76%
rename from gluten-data/src/main/java/org/apache/gluten/memory/nmm/ReservationListener.java
rename to gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListener.java
index f423bc8fb..6f8cb867e 100644
--- a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/ReservationListener.java
+++ b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListener.java
@@ -14,14 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.gluten.memory.nmm;
-
-import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
-import org.apache.gluten.memory.memtarget.NoopMemoryTarget;
+package org.apache.gluten.memory.listener;
public interface ReservationListener {
- ReservationListener NOOP =
- new ManagedReservationListener(new NoopMemoryTarget(), new SimpleMemoryUsageRecorder());
long reserve(long size);
diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java
new file mode 100644
index 000000000..47b9937eb
--- /dev/null
+++ b/gluten-data/src/main/java/org/apache/gluten/memory/listener/ReservationListeners.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.memory.listener;
+
+import org.apache.gluten.GlutenConfig;
+import org.apache.gluten.memory.MemoryUsageStatsBuilder;
+import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
+import org.apache.gluten.memory.memtarget.*;
+
+import org.apache.spark.memory.TaskMemoryManager;
+import org.apache.spark.util.TaskResources;
+
+import java.util.Collections;
+import java.util.Map;
+
+public final class ReservationListeners {
+ public static final ReservationListener NOOP =
+ new ManagedReservationListener(new NoopMemoryTarget(), new SimpleMemoryUsageRecorder());
+
+ public static ReservationListener create(
+ String name, Spiller spiller, Map<String, MemoryUsageStatsBuilder> mutableStats) {
+ if (!TaskResources.inSparkTask()) {
+ throw new IllegalStateException(
+ "Spillable reservation listener must be used in a Spark task.");
+ }
+ return create0(name, spiller, mutableStats);
+ }
+
+ private static ReservationListener create0(
+ String name, Spiller spiller, Map<String, MemoryUsageStatsBuilder> mutableStats) {
+ // Memory target.
+ final double overAcquiredRatio = GlutenConfig.getConf().memoryOverAcquiredRatio();
+ final long reservationBlockSize = GlutenConfig.getConf().memoryReservationBlockSize();
+ final TaskMemoryManager tmm = TaskResources.getLocalTaskContext().taskMemoryManager();
+ final MemoryTarget target =
+ MemoryTargets.throwOnOom(
+ MemoryTargets.overAcquire(
+ MemoryTargets.dynamicOffHeapSizingIfEnabled(
+ MemoryTargets.newConsumer(
+ tmm,
+ name,
+ Spillers.withMinSpillSize(spiller, reservationBlockSize),
+ mutableStats)),
+ MemoryTargets.dynamicOffHeapSizingIfEnabled(
+ MemoryTargets.newConsumer(
+ tmm,
+ "OverAcquire.DummyTarget",
+ new Spiller() {
+ @Override
+ public long spill(MemoryTarget self, Spiller.Phase phase, long size) {
+ if (!Spillers.PHASE_SET_ALL.contains(phase)) {
+ return 0L;
+ }
+ return self.repay(size);
+ }
+ },
+ Collections.emptyMap())),
+ overAcquiredRatio));
+
+ // Listener.
+ return new ManagedReservationListener(target, TaskResources.getSharedUsage());
+ }
+}
diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java b/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java
deleted file mode 100644
index 0d1a0c5ae..000000000
--- a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManager.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.memory.nmm;
-
-import org.apache.gluten.backendsapi.BackendsApiManager;
-import org.apache.gluten.memory.alloc.NativeMemoryAllocators;
-import org.apache.gluten.memory.memtarget.KnownNameAndStats;
-import org.apache.gluten.proto.MemoryUsageStats;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.spark.memory.SparkMemoryUtil;
-import org.apache.spark.util.TaskResource;
-import org.apache.spark.util.Utils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class NativeMemoryManager implements TaskResource {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(NativeMemoryManager.class);
-
- private final long nativeInstanceHandle;
- private final String name;
- private final ReservationListener listener;
-
- private NativeMemoryManager(
- String name, long nativeInstanceHandle, ReservationListener listener) {
- this.name = name;
- this.nativeInstanceHandle = nativeInstanceHandle;
- this.listener = listener;
- }
-
- public static NativeMemoryManager create(String name, ReservationListener listener) {
- long allocatorId = NativeMemoryAllocators.getDefault().get().getNativeInstanceId();
- return new NativeMemoryManager(
- name, create(BackendsApiManager.getBackendName(), name, allocatorId, listener), listener);
- }
-
- public long getNativeInstanceHandle() {
- return this.nativeInstanceHandle;
- }
-
- public MemoryUsageStats collectMemoryUsage() {
- try {
- return MemoryUsageStats.parseFrom(collectMemoryUsage(nativeInstanceHandle));
- } catch (InvalidProtocolBufferException e) {
- throw new RuntimeException(e);
- }
- }
-
- public long shrink(long size) {
- return shrink(nativeInstanceHandle, size);
- }
-
- // Hold this memory manager. The underlying memory pools will be released as lately as this
- // memory manager gets destroyed. Which means, a call to this function would make sure the
- // memory blocks directly or indirectly managed by this manager, be guaranteed safe to
- // access during the period that this manager is alive.
- public void hold() {
- hold(nativeInstanceHandle);
- }
-
- private static native long shrink(long memoryManagerId, long size);
-
- private static native long create(
- String backendType, String name, long allocatorId, ReservationListener listener);
-
- private static native void release(long memoryManagerId);
-
- private static native byte[] collectMemoryUsage(long memoryManagerId);
-
- private static native void hold(long memoryManagerId);
-
- @Override
- public void release() throws Exception {
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug(
- SparkMemoryUtil.prettyPrintStats(
- "About to release memory manager, usage dump:",
- new KnownNameAndStats() {
- @Override
- public String name() {
- return name;
- }
-
- @Override
- public MemoryUsageStats stats() {
- return collectMemoryUsage();
- }
- }));
- }
- release(nativeInstanceHandle);
- if (listener.getUsedBytes() != 0) {
- LOGGER.warn(
- String.format(
- "%s Reservation listener %s still reserved non-zero bytes, "
- + "which may cause memory leak, size: %s. ",
- name, listener.toString(), Utils.bytesToString(listener.getUsedBytes())));
- }
- }
-
- @Override
- public int priority() {
- return 0; // lowest release priority
- }
-
- @Override
- public String resourceName() {
- return name + "_mem";
- }
-}
diff --git a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManagers.java b/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManagers.java
deleted file mode 100644
index 37456badd..000000000
--- a/gluten-data/src/main/java/org/apache/gluten/memory/nmm/NativeMemoryManagers.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.memory.nmm;
-
-import org.apache.gluten.GlutenConfig;
-import org.apache.gluten.memory.MemoryUsageRecorder;
-import org.apache.gluten.memory.memtarget.MemoryTarget;
-import org.apache.gluten.memory.memtarget.MemoryTargets;
-import org.apache.gluten.memory.memtarget.Spiller;
-import org.apache.gluten.memory.memtarget.Spillers;
-import org.apache.gluten.proto.MemoryUsageStats;
-
-import org.apache.spark.memory.TaskMemoryManager;
-import org.apache.spark.util.TaskResources;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public final class NativeMemoryManagers {
- private static final Logger LOG = LoggerFactory.getLogger(NativeMemoryManagers.class);
-
- // TODO: Let all caller support spill.
- public static NativeMemoryManager contextInstance(String name) {
- if (!TaskResources.inSparkTask()) {
- throw new IllegalStateException("This method must be called in a Spark task.");
- }
- String id = "NativeMemoryManager:" + name;
- return TaskResources.addResourceIfNotRegistered(
- id, () -> createNativeMemoryManager(name, Collections.emptyList()));
- }
-
- public static NativeMemoryManager create(String name, Spiller... spillers) {
- if (!TaskResources.inSparkTask()) {
- throw new IllegalStateException("Spiller must be used in a Spark task.");
- }
-
- final NativeMemoryManager manager = createNativeMemoryManager(name, Arrays.asList(spillers));
- return TaskResources.addAnonymousResource(manager);
- }
-
- private static NativeMemoryManager createNativeMemoryManager(
- String name, List<Spiller> spillers) {
- final AtomicReference<NativeMemoryManager> out = new AtomicReference<>();
- // memory target
- final double overAcquiredRatio = GlutenConfig.getConf().memoryOverAcquiredRatio();
- final long reservationBlockSize = GlutenConfig.getConf().memoryReservationBlockSize();
- final TaskMemoryManager tmm = TaskResources.getLocalTaskContext().taskMemoryManager();
- final MemoryTarget target =
- MemoryTargets.throwOnOom(
- MemoryTargets.overAcquire(
- MemoryTargets.dynamicOffHeapSizingIfEnabled(
- MemoryTargets.newConsumer(
- tmm,
- name,
- // call memory manager's shrink API, if no good then call the spiller
- Stream.concat(
- Stream.of(
- new Spiller() {
- @Override
- public long spill(MemoryTarget self, long size) {
- return Optional.ofNullable(out.get())
- .map(nmm -> nmm.shrink(size))
- .orElseGet(
- () -> {
- LOG.warn(
- "Shrink is requested before native "
- + "memory manager is created. Try moving "
- + "any actions about memory allocation"
- + " out from the memory manager"
- + " constructor.");
- return 0L;
- });
- }
-
- @Override
- public Set<Phase> applicablePhases() {
- return Spillers.PHASE_SET_SHRINK_ONLY;
- }
- }),
- spillers.stream())
- .map(
- spiller -> Spillers.withMinSpillSize(spiller, reservationBlockSize))
- .collect(Collectors.toList()),
- Collections.singletonMap(
- "single",
- new MemoryUsageRecorder() {
- @Override
- public void inc(long bytes) {
- // no-op
- }
-
- @Override
- public long peak() {
- throw new UnsupportedOperationException("Not implemented");
- }
-
- @Override
- public long current() {
- throw new UnsupportedOperationException("Not implemented");
- }
-
- @Override
- public MemoryUsageStats toStats() {
- return getNativeMemoryManager().collectMemoryUsage();
- }
-
- private NativeMemoryManager getNativeMemoryManager() {
- return Optional.ofNullable(out.get())
- .orElseThrow(
- () ->
- new IllegalStateException(
- ""
- + "Memory usage stats are requested before"
- + " native memory manager is created. Try"
- + " moving any actions about memory"
- + " allocation out from the memory manager"
- + " constructor."));
- }
- }))),
- MemoryTargets.dynamicOffHeapSizingIfEnabled(
- MemoryTargets.newConsumer(
- tmm,
- "OverAcquire.DummyTarget",
- Collections.singletonList(
- new Spiller() {
- @Override
- public long spill(MemoryTarget self, long size) {
- return self.repay(size);
- }
-
- @Override
- public Set<Phase> applicablePhases() {
- return Spillers.PHASE_SET_ALL;
- }
- }),
- Collections.emptyMap())),
- overAcquiredRatio));
- // listener
- ManagedReservationListener rl =
- new ManagedReservationListener(target, TaskResources.getSharedUsage());
- // native memory manager
- out.set(NativeMemoryManager.create(name, rl));
- return out.get();
- }
-}
diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java
index bd89f62a1..c147862d0 100644
--- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java
+++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchInIterator.java
@@ -18,6 +18,7 @@ package org.apache.gluten.vectorized;
import org.apache.gluten.columnarbatch.ColumnarBatchJniWrapper;
import org.apache.gluten.columnarbatch.ColumnarBatches;
+import org.apache.gluten.exec.Runtimes;
import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators;
import org.apache.spark.sql.vectorized.ColumnarBatch;
@@ -25,6 +26,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch;
import java.util.Iterator;
public class ColumnarBatchInIterator extends GeneralInIterator {
+
public ColumnarBatchInIterator(Iterator<ColumnarBatch> delegated) {
super(delegated);
}
@@ -33,7 +35,8 @@ public class ColumnarBatchInIterator extends GeneralInIterator {
final ColumnarBatch next = nextColumnarBatch();
if (next.numCols() == 0) {
// the operation will find a zero column batch from a task-local pool
- return ColumnarBatchJniWrapper.create().getForEmptySchema(next.numRows());
+ return ColumnarBatchJniWrapper.create(Runtimes.contextInstance("ColumnarBatchInIterator"))
+ .getForEmptySchema(next.numRows());
}
final ColumnarBatch offloaded =
ColumnarBatches.ensureOffloaded(ArrowBufferAllocators.contextInstance(), next);
diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java
index 3a2a741be..ddf00844f 100644
--- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java
+++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchOutIterator.java
@@ -19,7 +19,6 @@ package org.apache.gluten.vectorized;
import org.apache.gluten.columnarbatch.ColumnarBatches;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.RuntimeAware;
-import org.apache.gluten.memory.nmm.NativeMemoryManager;
import org.apache.gluten.metrics.IMetrics;
import org.apache.spark.sql.vectorized.ColumnarBatch;
@@ -29,13 +28,11 @@ import java.io.IOException;
public class ColumnarBatchOutIterator extends GeneralOutIterator implements RuntimeAware {
private final Runtime runtime;
private final long iterHandle;
- private final NativeMemoryManager nmm;
- public ColumnarBatchOutIterator(Runtime runtime, long iterHandle, NativeMemoryManager nmm) {
+ public ColumnarBatchOutIterator(Runtime runtime, long iterHandle) {
super();
this.runtime = runtime;
this.iterHandle = iterHandle;
- this.nmm = nmm;
}
@Override
@@ -88,8 +85,9 @@ public class ColumnarBatchOutIterator extends GeneralOutIterator implements Runt
@Override
public void closeInternal() {
- nmm.hold(); // to make sure the outputted batches are still accessible after the iterator is
- // closed
+ // To make sure the outputted batches are still accessible after the iterator is closed.
+ // TODO: Remove this API if we have other choice, e.g., hold the pools in native code.
+ runtime.holdMemory();
nativeClose(iterHandle);
}
}
diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java
index 59f31689e..bfe0d7561 100644
--- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java
+++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ColumnarBatchSerializerJniWrapper.java
@@ -18,7 +18,6 @@ package org.apache.gluten.vectorized;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.RuntimeAware;
-import org.apache.gluten.exec.Runtimes;
public class ColumnarBatchSerializerJniWrapper implements RuntimeAware {
private final Runtime runtime;
@@ -27,11 +26,7 @@ public class ColumnarBatchSerializerJniWrapper implements RuntimeAware {
this.runtime = runtime;
}
- public static ColumnarBatchSerializerJniWrapper create() {
- return new ColumnarBatchSerializerJniWrapper(Runtimes.contextInstance());
- }
-
- public static ColumnarBatchSerializerJniWrapper forRuntime(Runtime runtime) {
+ public static ColumnarBatchSerializerJniWrapper create(Runtime runtime) {
return new ColumnarBatchSerializerJniWrapper(runtime);
}
@@ -40,10 +35,10 @@ public class ColumnarBatchSerializerJniWrapper implements RuntimeAware {
return runtime.getHandle();
}
- public native ColumnarBatchSerializeResult serialize(long[] handles, long memoryManagerHandle);
+ public native ColumnarBatchSerializeResult serialize(long[] handles);
// Return the native ColumnarBatchSerializer handle
- public native long init(long cSchema, long memoryManagerHandle);
+ public native long init(long cSchema);
public native long deserialize(long serializerHandle, byte[] data);
diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java
index 7ae2fafb3..7f8de78f9 100644
--- a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java
+++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeColumnarToRowJniWrapper.java
@@ -18,7 +18,6 @@ package org.apache.gluten.vectorized;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.RuntimeAware;
-import org.apache.gluten.exec.Runtimes;
public class NativeColumnarToRowJniWrapper implements RuntimeAware {
private final Runtime runtime;
@@ -27,8 +26,8 @@ public class NativeColumnarToRowJniWrapper implements RuntimeAware {
this.runtime = runtime;
}
- public static NativeColumnarToRowJniWrapper create() {
- return new NativeColumnarToRowJniWrapper(Runtimes.contextInstance());
+ public static NativeColumnarToRowJniWrapper create(Runtime runtime) {
+ return new NativeColumnarToRowJniWrapper(runtime);
}
@Override
@@ -36,9 +35,9 @@ public class NativeColumnarToRowJniWrapper implements RuntimeAware {
return runtime.getHandle();
}
- public native long nativeColumnarToRowInit(long memoryManagerHandle) throws RuntimeException;
+ public native long nativeColumnarToRowInit() throws RuntimeException;
- public native NativeColumnarToRowInfo nativeColumnarToRowConvert(long batchHandle, long c2rHandle)
+ public native NativeColumnarToRowInfo nativeColumnarToRowConvert(long c2rHandle, long batchHandle)
throws RuntimeException;
public native void nativeClose(long c2rHandle);
diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
index 2ac048b2b..e5eea029b 100644
--- a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
+++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
@@ -22,8 +22,6 @@ import org.apache.gluten.exec.Runtimes;
import org.apache.gluten.memory.memtarget.MemoryTarget;
import org.apache.gluten.memory.memtarget.Spiller;
import org.apache.gluten.memory.memtarget.Spillers;
-import org.apache.gluten.memory.nmm.NativeMemoryManager;
-import org.apache.gluten.memory.nmm.NativeMemoryManagers;
import org.apache.gluten.utils.DebugUtil;
import org.apache.gluten.validate.NativePlanValidationInfo;
@@ -33,17 +31,15 @@ import org.apache.spark.util.SparkDirectoryUtil;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
-import java.util.Optional;
-import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.atomic.AtomicReference;
public class NativePlanEvaluator {
+ private final Runtime runtime = Runtimes.contextInstance("WholeStageIterator");
private final PlanEvaluatorJniWrapper jniWrapper;
private NativePlanEvaluator() {
- jniWrapper = PlanEvaluatorJniWrapper.create();
+ jniWrapper = PlanEvaluatorJniWrapper.create(runtime);
}
public static NativePlanEvaluator create() {
@@ -63,31 +59,6 @@ public class NativePlanEvaluator {
public GeneralOutIterator createKernelWithBatchIterator(
byte[] wsPlan, byte[][] splitInfo, List<GeneralInIterator> iterList, int partitionIndex)
throws RuntimeException, IOException {
- final AtomicReference<ColumnarBatchOutIterator> outIterator = new AtomicReference<>();
- final NativeMemoryManager nmm =
- NativeMemoryManagers.create(
- "WholeStageIterator",
- new Spiller() {
- @Override
- public long spill(MemoryTarget self, long size) {
- ColumnarBatchOutIterator instance =
- Optional.ofNullable(outIterator.get())
- .orElseThrow(
- () ->
- new IllegalStateException(
- "Fatal: spill() called before a output iterator "
- + "is created. This behavior should be optimized "
- + "by moving memory allocations from create() to "
- + "hasNext()/next()"));
- return instance.spill(size);
- }
-
- @Override
- public Set<Phase> applicablePhases() {
- return Spillers.PHASE_SET_SPILL_ONLY;
- }
- });
- final long memoryManagerHandle = nmm.getNativeInstanceHandle();
final String spillDirPath =
SparkDirectoryUtil.get()
@@ -95,9 +66,8 @@ public class NativePlanEvaluator {
.mkChildDirRoundRobin(UUID.randomUUID().toString())
.getAbsolutePath();
- long iterHandle =
+ final long itrHandle =
jniWrapper.nativeCreateKernelWithIterator(
- memoryManagerHandle,
wsPlan,
splitInfo,
iterList.toArray(new GeneralInIterator[0]),
@@ -106,12 +76,21 @@ public class NativePlanEvaluator {
TaskContext.get().taskAttemptId(),
DebugUtil.saveInputToFile(),
BackendsApiManager.getSparkPlanExecApiInstance().rewriteSpillPath(spillDirPath));
- outIterator.set(createOutIterator(Runtimes.contextInstance(), iterHandle, nmm));
- return outIterator.get();
+ final ColumnarBatchOutIterator out = createOutIterator(runtime, itrHandle);
+ runtime.addSpiller(
+ new Spiller() {
+ @Override
+ public long spill(MemoryTarget self, Spiller.Phase phase, long size) {
+ if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) {
+ return 0L;
+ }
+ return out.spill(size);
+ }
+ });
+ return out;
}
- private ColumnarBatchOutIterator createOutIterator(
- Runtime runtime, long iterHandle, NativeMemoryManager nmm) throws IOException {
- return new ColumnarBatchOutIterator(runtime, iterHandle, nmm);
+ private ColumnarBatchOutIterator createOutIterator(Runtime runtime, long itrHandle) {
+ return new ColumnarBatchOutIterator(runtime, itrHandle);
}
}
diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java
index b43c92b99..1185d52fe 100644
--- a/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java
+++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/NativeRowToColumnarJniWrapper.java
@@ -18,7 +18,6 @@ package org.apache.gluten.vectorized;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.RuntimeAware;
-import org.apache.gluten.exec.Runtimes;
public class NativeRowToColumnarJniWrapper implements RuntimeAware {
private final Runtime runtime;
@@ -27,8 +26,8 @@ public class NativeRowToColumnarJniWrapper implements RuntimeAware {
this.runtime = runtime;
}
- public static NativeRowToColumnarJniWrapper create() {
- return new NativeRowToColumnarJniWrapper(Runtimes.contextInstance());
+ public static NativeRowToColumnarJniWrapper create(Runtime runtime) {
+ return new NativeRowToColumnarJniWrapper(runtime);
}
@Override
@@ -36,7 +35,7 @@ public class NativeRowToColumnarJniWrapper implements RuntimeAware {
return runtime.getHandle();
}
- public native long init(long cSchema, long memoryManagerHandle);
+ public native long init(long cSchema);
public native long nativeConvertRowToColumnar(
long r2cHandle, long[] rowLength, long bufferAddress);
diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java
index 5e8ae392a..eecd7c9e2 100644
--- a/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java
+++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/PlanEvaluatorJniWrapper.java
@@ -18,7 +18,6 @@ package org.apache.gluten.vectorized;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.RuntimeAware;
-import org.apache.gluten.exec.Runtimes;
import org.apache.gluten.validate.NativePlanValidationInfo;
/**
@@ -33,8 +32,8 @@ public class PlanEvaluatorJniWrapper implements RuntimeAware {
this.runtime = runtime;
}
- public static PlanEvaluatorJniWrapper create() {
- return new PlanEvaluatorJniWrapper(Runtimes.contextInstance());
+ public static PlanEvaluatorJniWrapper create(Runtime runtime) {
+ return new PlanEvaluatorJniWrapper(runtime);
}
@Override
@@ -57,11 +56,9 @@ public class PlanEvaluatorJniWrapper implements RuntimeAware {
/**
* Create a native compute kernel and return a columnar result iterator.
*
- * @param memoryManagerHandle NativeMemoryManager instance handle
* @return iterator instance id
*/
public native long nativeCreateKernelWithIterator(
- long memoryManagerHandle,
byte[] wsPlan,
byte[][] splitInfo,
GeneralInIterator[] batchItr,
diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java
index 24425ccf7..515486e45 100644
--- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java
+++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleReaderJniWrapper.java
@@ -18,7 +18,6 @@ package org.apache.gluten.vectorized;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.RuntimeAware;
-import org.apache.gluten.exec.Runtimes;
public class ShuffleReaderJniWrapper implements RuntimeAware {
private final Runtime runtime;
@@ -27,8 +26,8 @@ public class ShuffleReaderJniWrapper implements RuntimeAware {
this.runtime = runtime;
}
- public static ShuffleReaderJniWrapper create() {
- return new ShuffleReaderJniWrapper(Runtimes.contextInstance());
+ public static ShuffleReaderJniWrapper create(Runtime runtime) {
+ return new ShuffleReaderJniWrapper(runtime);
}
@Override
@@ -38,7 +37,6 @@ public class ShuffleReaderJniWrapper implements RuntimeAware {
public native long make(
long cSchema,
- long memoryManagerHandle,
String compressionType,
String compressionCodecBackend,
int batchSize,
diff --git a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java
index f4e117275..883fc6001 100644
--- a/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java
+++ b/gluten-data/src/main/java/org/apache/gluten/vectorized/ShuffleWriterJniWrapper.java
@@ -18,7 +18,6 @@ package org.apache.gluten.vectorized;
import org.apache.gluten.exec.Runtime;
import org.apache.gluten.exec.RuntimeAware;
-import org.apache.gluten.exec.Runtimes;
import java.io.IOException;
@@ -29,8 +28,8 @@ public class ShuffleWriterJniWrapper implements RuntimeAware {
this.runtime = runtime;
}
- public static ShuffleWriterJniWrapper create() {
- return new ShuffleWriterJniWrapper(Runtimes.contextInstance());
+ public static ShuffleWriterJniWrapper create(Runtime runtime) {
+ return new ShuffleWriterJniWrapper(runtime);
}
@Override
@@ -65,7 +64,6 @@ public class ShuffleWriterJniWrapper implements RuntimeAware {
String dataFile,
int subDirsPerLocalDir,
String localDirs,
- long memoryManagerHandle,
double reallocThreshold,
long handle,
long taskAttemptId,
@@ -85,7 +83,6 @@ public class ShuffleWriterJniWrapper implements RuntimeAware {
dataFile,
subDirsPerLocalDir,
localDirs,
- memoryManagerHandle,
reallocThreshold,
handle,
taskAttemptId,
@@ -115,7 +112,6 @@ public class ShuffleWriterJniWrapper implements RuntimeAware {
int pushBufferMaxSize,
long sortBufferMaxSize,
Object pusher,
- long memoryManagerHandle,
long handle,
long taskAttemptId,
int startPartitionId,
@@ -136,7 +132,6 @@ public class ShuffleWriterJniWrapper implements RuntimeAware {
null,
0,
null,
- memoryManagerHandle,
reallocThreshold,
handle,
taskAttemptId,
@@ -162,7 +157,6 @@ public class ShuffleWriterJniWrapper implements RuntimeAware {
String dataFile,
int subDirsPerLocalDir,
String localDirs,
- long memoryManagerHandle,
double reallocThreshold,
long handle,
long taskAttemptId,
diff --git a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtime.scala b/gluten-data/src/main/scala/org/apache/gluten/exec/Runtime.scala
index 12d855c71..1f632659e 100644
--- a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtime.scala
+++ b/gluten-data/src/main/scala/org/apache/gluten/exec/Runtime.scala
@@ -20,35 +20,111 @@ import org.apache.gluten.GlutenConfig
import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.exception.GlutenException
import org.apache.gluten.init.JniUtils
+import org.apache.gluten.memory.MemoryUsageStatsBuilder
+import org.apache.gluten.memory.listener.ReservationListeners
+import org.apache.gluten.memory.memtarget.{KnownNameAndStats, MemoryTarget, Spiller, Spillers}
+import org.apache.gluten.proto.MemoryUsageStats
-import org.apache.spark.sql.internal.GlutenConfigUtil
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.memory.SparkMemoryUtil
+import org.apache.spark.sql.internal.{GlutenConfigUtil, SQLConf}
import org.apache.spark.util.TaskResource
+import org.slf4j.LoggerFactory
+
import java.util.concurrent.atomic.AtomicBoolean
-class Runtime private[exec] () extends TaskResource {
- private val handle = RuntimeJniWrapper.createRuntime(
- BackendsApiManager.getBackendName,
- JniUtils.toNativeConf(
- GlutenConfig.getNativeSessionConf(
- BackendsApiManager.getSettings.getBackendConfigPrefix,
- GlutenConfigUtil.parseConfig(SQLConf.get.getAllConfs)))
- )
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+trait Runtime {
+ def addSpiller(spiller: Spiller): Unit
+ def holdMemory(): Unit
+ def collectMemoryUsage(): MemoryUsageStats
+ def getHandle(): Long
+}
+
+object Runtime {
+ private[exec] def apply(name: String): Runtime with TaskResource = {
+ new RuntimeImpl(name)
+ }
+
+ private class RuntimeImpl(name: String) extends Runtime with TaskResource {
+ private val LOGGER = LoggerFactory.getLogger(classOf[Runtime])
+
+ private val spillers = Spillers.appendable()
+ private val mutableStats: mutable.Map[String, MemoryUsageStatsBuilder] = mutable.Map()
+ private val rl = ReservationListeners.create(resourceName(), spillers, mutableStats.asJava)
+ private val handle = RuntimeJniWrapper.createRuntime(
+ BackendsApiManager.getBackendName,
+ rl,
+ JniUtils.toNativeConf(
+ GlutenConfig.getNativeSessionConf(
+ BackendsApiManager.getSettings.getBackendConfigPrefix,
+ GlutenConfigUtil.parseConfig(SQLConf.get.getAllConfs)))
+ )
- private val released: AtomicBoolean = new AtomicBoolean(false)
+ spillers.append(new Spiller() {
+ override def spill(self: MemoryTarget, phase: Spiller.Phase, size: Long): Long = {
+ if (!Spillers.PHASE_SET_SHRINK_ONLY.contains(phase)) {
+ // Only respond for shrinking.
+ return 0L
+ }
+ RuntimeJniWrapper.shrinkMemory(handle, size)
+ }
+ })
+ mutableStats += "single" -> new MemoryUsageStatsBuilder {
+ override def toStats: MemoryUsageStats = collectMemoryUsage()
+ }
- def getHandle: Long = handle
+ private val released: AtomicBoolean = new AtomicBoolean(false)
- override def release(): Unit = {
- if (!released.compareAndSet(false, true)) {
- throw new GlutenException(
- s"Runtime instance already released: $getHandle, ${resourceName()}, ${priority()}")
+ def getHandle: Long = handle
+
+ def addSpiller(spiller: Spiller): Unit = {
+ spillers.append(spiller)
}
- RuntimeJniWrapper.releaseRuntime(handle)
- }
- override def priority(): Int = 10
+ def holdMemory(): Unit = {
+ RuntimeJniWrapper.holdMemory(handle)
+ }
+
+ def collectMemoryUsage(): MemoryUsageStats = {
+ MemoryUsageStats.parseFrom(RuntimeJniWrapper.collectMemoryUsage(handle))
+ }
+
+ override def release(): Unit = {
+ if (!released.compareAndSet(false, true)) {
+ throw new GlutenException(
+ s"Runtime instance already released: $handle, ${resourceName()}, ${priority()}")
+ }
+ if (LOGGER.isDebugEnabled) {
+ LOGGER.debug(
+ SparkMemoryUtil.prettyPrintStats(
+ "About to release memory manager, usage dump:",
+ new KnownNameAndStats() {
+ override def name: String = resourceName()
+
+ override def stats: MemoryUsageStats = collectMemoryUsage()
+ }
+ ))
+ }
- override def resourceName(): String = s"Runtime_" + handle
+ RuntimeJniWrapper.releaseRuntime(handle)
+
+ if (rl.getUsedBytes != 0) {
+ LOGGER.warn(
+ String.format(
+ "%s Reservation listener %s still reserved non-zero bytes, which may cause memory" +
+ " leak, size: %s. ",
+ name,
+ rl.toString,
+ SparkMemoryUtil.bytesToString(rl.getUsedBytes)
+ ))
+ }
+ }
+
+ override def priority(): Int = 0
+
+ override def resourceName(): String = name
+ }
}
diff --git a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtimes.scala b/gluten-data/src/main/scala/org/apache/gluten/exec/Runtimes.scala
index e5204771c..3614fe05f 100644
--- a/gluten-data/src/main/scala/org/apache/gluten/exec/Runtimes.scala
+++ b/gluten-data/src/main/scala/org/apache/gluten/exec/Runtimes.scala
@@ -16,21 +16,20 @@
*/
package org.apache.gluten.exec
-import org.apache.spark.util.TaskResources
+import org.apache.spark.util.{TaskResource, TaskResources}
object Runtimes {
- private val RUNTIME_NAME = "Runtime"
/** Get or create the runtime which bound with Spark TaskContext. */
- def contextInstance(): Runtime = {
+ def contextInstance(name: String): Runtime = {
if (!TaskResources.inSparkTask()) {
throw new IllegalStateException("This method must be called in a Spark task.")
}
- TaskResources.addResourceIfNotRegistered(RUNTIME_NAME, () => create())
+ TaskResources.addResourceIfNotRegistered(name, () => create(name))
}
- private def create(): Runtime = {
- new Runtime
+ private def create(name: String): Runtime with TaskResource = {
+ Runtime(name)
}
}
diff --git a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala
index 326f836a0..e75abe41e 100644
--- a/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala
+++ b/gluten-data/src/main/scala/org/apache/gluten/vectorized/ColumnarBatchSerializer.scala
@@ -19,7 +19,6 @@ package org.apache.gluten.vectorized
import org.apache.gluten.GlutenConfig
import org.apache.gluten.exec.Runtimes
import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.ArrowAbiUtil
import org.apache.spark.SparkEnv
@@ -79,7 +78,6 @@ private class ColumnarBatchSerializerInstance(
extends SerializerInstance
with Logging {
- private val nmm = NativeMemoryManagers.contextInstance("ShuffleReader")
private val shuffleReaderHandle = {
val allocator: BufferAllocator = ArrowBufferAllocators
.contextInstance(classOf[ColumnarBatchSerializerInstance].getSimpleName)
@@ -98,15 +96,14 @@ private class ColumnarBatchSerializerInstance(
val compressionCodecBackend =
GlutenConfig.getConf.columnarShuffleCodecBackend.orNull
val batchSize = GlutenConfig.getConf.maxBatchSize
- val jniWrapper = ShuffleReaderJniWrapper.create()
+ val runtime = Runtimes.contextInstance("ShuffleReader")
+ val jniWrapper = ShuffleReaderJniWrapper.create(runtime)
val shuffleReaderHandle = jniWrapper.make(
cSchema.memoryAddress(),
- nmm.getNativeInstanceHandle,
compressionCodec,
compressionCodecBackend,
batchSize,
- shuffleWriterType
- )
+ shuffleWriterType)
// Close shuffle reader instance as lately as the end of task processing,
// since the native reader could hold a reference to memory pool that
// was used to create all buffers read from shuffle reader. The pool
@@ -137,12 +134,12 @@ private class ColumnarBatchSerializerInstance(
extends DeserializationStream
with TaskResource {
private val byteIn: JniByteInputStream = JniByteInputStreams.create(in)
+ private val runtime = Runtimes.contextInstance("ShuffleReader")
private val wrappedOut: GeneralOutIterator = new ColumnarBatchOutIterator(
- Runtimes.contextInstance(),
+ runtime,
ShuffleReaderJniWrapper
- .create()
- .readStream(shuffleReaderHandle, byteIn),
- nmm)
+ .create(runtime)
+ .readStream(shuffleReaderHandle, byteIn))
private var cb: ColumnarBatch = _
diff --git a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala
index af004b54f..4e59d0ba6 100644
--- a/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala
+++ b/gluten-data/src/main/scala/org/apache/spark/shuffle/ColumnarShuffleWriter.scala
@@ -18,10 +18,8 @@ package org.apache.spark.shuffle
import org.apache.gluten.GlutenConfig
import org.apache.gluten.columnarbatch.ColumnarBatches
-import org.apache.gluten.memory.memtarget.MemoryTarget
-import org.apache.gluten.memory.memtarget.Spiller
-import org.apache.gluten.memory.memtarget.Spillers
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
+import org.apache.gluten.exec.Runtimes
+import org.apache.gluten.memory.memtarget.{MemoryTarget, Spiller, Spillers}
import org.apache.gluten.vectorized._
import org.apache.spark._
@@ -98,7 +96,9 @@ class ColumnarShuffleWriter[K, V](
private val reallocThreshold = GlutenConfig.getConf.columnarShuffleReallocThreshold
- private val jniWrapper = ShuffleWriterJniWrapper.create()
+ private val runtime = Runtimes.contextInstance("ShuffleWriter")
+
+ private val jniWrapper = ShuffleWriterJniWrapper.create(runtime)
private var nativeShuffleWriter: Long = -1L
@@ -153,36 +153,25 @@ class ColumnarShuffleWriter[K, V](
dataTmp.getAbsolutePath,
blockManager.subDirsPerLocalDir,
localDirs,
- NativeMemoryManagers
- .create(
- "ShuffleWriter",
- new Spiller() {
- override def spill(self: MemoryTarget, size: Long): Long = {
- if (nativeShuffleWriter == -1L) {
- throw new IllegalStateException(
- "Fatal: spill() called before a shuffle writer " +
- "is created. This behavior should be optimized by moving memory " +
- "allocations from make() to split()")
- }
- logInfo(s"Gluten shuffle writer: Trying to spill $size bytes of data")
- // fixme pass true when being called by self
- val spilled =
- jniWrapper.nativeEvict(nativeShuffleWriter, size, false)
- logInfo(s"Gluten shuffle writer: Spilled $spilled / $size bytes of data")
- spilled
- }
-
- override def applicablePhases(): java.util.Set[Spiller.Phase] =
- Spillers.PHASE_SET_SPILL_ONLY
- }
- )
- .getNativeInstanceHandle,
reallocThreshold,
handle,
taskContext.taskAttemptId(),
GlutenShuffleUtils.getStartPartitionId(dep.nativePartitioning, taskContext.partitionId),
shuffleWriterType
)
+ runtime.addSpiller(new Spiller() {
+ override def spill(self: MemoryTarget, phase: Spiller.Phase, size: Long): Long = {
+ if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) {
+ return 0L
+ }
+ logInfo(s"Gluten shuffle writer: Trying to spill $size bytes of data")
+ // fixme pass true when being called by self
+ val spilled =
+ jniWrapper.nativeEvict(nativeShuffleWriter, size, false)
+ logInfo(s"Gluten shuffle writer: Spilled $spilled / $size bytes of data")
+ spilled
+ }
+ })
}
val startTime = System.nanoTime()
jniWrapper.write(nativeShuffleWriter, rows, handle, availableOffHeapPerTask())
diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
index 840f8618b..2f6abdc37 100644
--- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
+++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/ColumnarBuildSideRelation.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.execution
import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.exec.Runtimes
import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.sql.shims.SparkShimLoader
import org.apache.gluten.utils.ArrowAbiUtil
import org.apache.gluten.utils.iterator.Iterators
@@ -41,7 +40,8 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra
extends BuildSideRelation {
override def deserialized: Iterator[ColumnarBatch] = {
- val jniWrapper = ColumnarBatchSerializerJniWrapper.create()
+ val runtime = Runtimes.contextInstance("BuildSideRelation#deserialized")
+ val jniWrapper = ColumnarBatchSerializerJniWrapper.create(runtime)
val serializeHandle: Long = {
val allocator = ArrowBufferAllocators.contextInstance()
val cSchema = ArrowSchema.allocateNew(allocator)
@@ -50,11 +50,7 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra
SQLConf.get.sessionLocalTimeZone)
ArrowAbiUtil.exportSchema(allocator, arrowSchema, cSchema)
val handle = jniWrapper
- .init(
- cSchema.memoryAddress(),
- NativeMemoryManagers
- .contextInstance("BuildSideRelation#BatchSerializer")
- .getNativeInstanceHandle)
+ .init(cSchema.memoryAddress())
cSchema.close()
handle
}
@@ -72,7 +68,7 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra
jniWrapper
.deserialize(serializeHandle, batches(batchId))
batchId += 1
- ColumnarBatches.create(Runtimes.contextInstance(), handle)
+ ColumnarBatches.create(runtime, handle)
}
})
.protectInvocationFlow()
@@ -90,10 +86,9 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra
* was called in Spark Driver, should manage resources carefully.
*/
override def transform(key: Expression): Array[InternalRow] = TaskResources.runUnsafe {
+ val runtime = Runtimes.contextInstance("BuildSideRelation#transform")
// This transformation happens in Spark driver, thus resources can not be managed automatically.
- val runtime = Runtimes.contextInstance()
- val nativeMemoryManager = NativeMemoryManagers.contextInstance("BuildSideRelation#transform")
- val serializerJniWrapper = ColumnarBatchSerializerJniWrapper.create()
+ val serializerJniWrapper = ColumnarBatchSerializerJniWrapper.create(runtime)
val serializeHandle = {
val allocator = ArrowBufferAllocators.contextInstance()
val cSchema = ArrowSchema.allocateNew(allocator)
@@ -101,8 +96,7 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra
SparkShimLoader.getSparkShims.structFromAttributes(output),
SQLConf.get.sessionLocalTimeZone)
ArrowAbiUtil.exportSchema(allocator, arrowSchema, cSchema)
- val handle = serializerJniWrapper
- .init(cSchema.memoryAddress(), nativeMemoryManager.getNativeInstanceHandle)
+ val handle = serializerJniWrapper.init(cSchema.memoryAddress())
cSchema.close()
handle
}
@@ -110,8 +104,8 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra
var closed = false
// Convert columnar to Row.
- val jniWrapper = NativeColumnarToRowJniWrapper.create()
- val c2rId = jniWrapper.nativeColumnarToRowInit(nativeMemoryManager.getNativeInstanceHandle)
+ val jniWrapper = NativeColumnarToRowJniWrapper.create(runtime)
+ val c2rId = jniWrapper.nativeColumnarToRowInit()
var batchId = 0
val iterator = if (batches.length > 0) {
val res: Iterator[Iterator[InternalRow]] = new Iterator[Iterator[InternalRow]] {
@@ -142,7 +136,7 @@ case class ColumnarBuildSideRelation(output: Seq[Attribute], batches: Array[Arra
val cols = batch.numCols()
val rows = batch.numRows()
val info =
- jniWrapper.nativeColumnarToRowConvert(batchHandle, c2rId)
+ jniWrapper.nativeColumnarToRowConvert(c2rId, ColumnarBatches.getNativeHandle(batch))
batch.close()
val columnNames = key.flatMap {
case expression: AttributeReference =>
diff --git a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala
index 090b8fa25..22b376a1b 100644
--- a/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala
+++ b/gluten-data/src/main/scala/org/apache/spark/sql/execution/utils/ExecUtil.scala
@@ -17,8 +17,8 @@
package org.apache.spark.sql.execution.utils
import org.apache.gluten.columnarbatch.ColumnarBatches
+import org.apache.gluten.exec.Runtimes
import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
-import org.apache.gluten.memory.nmm.NativeMemoryManagers
import org.apache.gluten.utils.iterator.Iterators
import org.apache.gluten.vectorized.{ArrowWritableColumnVector, NativeColumnarToRowInfo, NativeColumnarToRowJniWrapper, NativePartitioning}
@@ -41,14 +41,12 @@ import org.apache.spark.util.MutablePair
object ExecUtil {
def convertColumnarToRow(batch: ColumnarBatch): Iterator[InternalRow] = {
- val jniWrapper = NativeColumnarToRowJniWrapper.create()
+ val runtime = Runtimes.contextInstance("ExecUtil#ColumnarToRow")
+ val jniWrapper = NativeColumnarToRowJniWrapper.create(runtime)
var info: NativeColumnarToRowInfo = null
val batchHandle = ColumnarBatches.getNativeHandle(batch)
- val c2rHandle = jniWrapper.nativeColumnarToRowInit(
- NativeMemoryManagers
- .contextInstance("ExecUtil#ColumnarToRow")
- .getNativeInstanceHandle)
- info = jniWrapper.nativeColumnarToRowConvert(batchHandle, c2rHandle)
+ val c2rHandle = jniWrapper.nativeColumnarToRowInit()
+ info = jniWrapper.nativeColumnarToRowConvert(c2rHandle, batchHandle)
Iterators
.wrap(new Iterator[InternalRow] {
@@ -147,7 +145,10 @@ object ExecUtil {
val newHandle = ColumnarBatches.compose(pidBatch, cb)
// Composed batch already hold pidBatch's shared ref, so close is safe.
ColumnarBatches.forceClose(pidBatch)
- (0, ColumnarBatches.create(ColumnarBatches.getRuntime(cb), newHandle))
+ (
+ 0,
+ ColumnarBatches
+ .create(Runtimes.contextInstance("ExecUtil#getShuffleDependency"), newHandle))
})
.recyclePayload(p => ColumnarBatches.forceClose(p._2)) // FIXME why force close?
.create()
diff --git a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java
index 471d70f4d..bd205ba7a 100644
--- a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java
+++ b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java
@@ -18,10 +18,11 @@ package org.apache.spark.shuffle.writer;
import org.apache.gluten.GlutenConfig;
import org.apache.gluten.columnarbatch.ColumnarBatches;
+import org.apache.gluten.exec.Runtime;
+import org.apache.gluten.exec.Runtimes;
import org.apache.gluten.memory.memtarget.MemoryTarget;
import org.apache.gluten.memory.memtarget.Spiller;
import org.apache.gluten.memory.memtarget.Spillers;
-import org.apache.gluten.memory.nmm.NativeMemoryManagers;
import org.apache.gluten.vectorized.ShuffleWriterJniWrapper;
import org.apache.gluten.vectorized.SplitResult;
@@ -45,7 +46,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
-import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
@@ -67,7 +67,8 @@ public class VeloxUniffleColumnarShuffleWriter<K, V> extends RssShuffleWriter<K,
private int compressionLevel;
private int partitionId;
- private ShuffleWriterJniWrapper jniWrapper = ShuffleWriterJniWrapper.create();
+ private Runtime runtime = Runtimes.contextInstance("UniffleShuffleWriter");
+ private ShuffleWriterJniWrapper jniWrapper = ShuffleWriterJniWrapper.create(runtime);
private SplitResult splitResult;
private int nativeBufferSize = GlutenConfig.getConf().maxBatchSize();
private int bufferSize;
@@ -150,35 +151,6 @@ public class VeloxUniffleColumnarShuffleWriter<K, V> extends RssShuffleWriter<K,
bufferSize,
bufferSize,
partitionPusher,
- NativeMemoryManagers.create(
- "UniffleShuffleWriter",
- new Spiller() {
- @Override
- public long spill(MemoryTarget self, long size) {
- if (nativeShuffleWriter == -1) {
- throw new IllegalStateException(
- "Fatal: spill() called before a shuffle shuffle writer "
- + "evaluator is created. This behavior should be"
- + "optimized by moving memory "
- + "allocations from make() to split()");
- }
- LOG.info(
- "Gluten shuffle writer: Trying to push {} bytes of data", size);
- long pushed =
- jniWrapper.nativeEvict(nativeShuffleWriter, size, false);
- LOG.info(
- "Gluten shuffle writer: Pushed {} / {} bytes of data",
- pushed,
- size);
- return pushed;
- }
-
- @Override
- public Set<Phase> applicablePhases() {
- return Spillers.PHASE_SET_SPILL_ONLY;
- }
- })
- .getNativeInstanceHandle(),
handle,
taskAttemptId,
GlutenShuffleUtils.getStartPartitionId(
@@ -186,6 +158,26 @@ public class VeloxUniffleColumnarShuffleWriter<K, V> extends RssShuffleWriter<K,
"uniffle",
isSort ? "sort" : "hash",
reallocThreshold);
+ runtime.addSpiller(
+ new Spiller() {
+ @Override
+ public long spill(MemoryTarget self, Spiller.Phase phase, long size) {
+ if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) {
+ return 0L;
+ }
+ if (nativeShuffleWriter == -1) {
+ throw new IllegalStateException(
+ "Fatal: spill() called before a shuffle shuffle writer "
+ + "evaluator is created. This behavior should be"
+ + "optimized by moving memory "
+ + "allocations from make() to split()");
+ }
+ LOG.info("Gluten shuffle writer: Trying to push {} bytes of data", size);
+ long pushed = jniWrapper.nativeEvict(nativeShuffleWriter, size, false);
+ LOG.info("Gluten shuffle writer: Pushed {} / {} bytes of data", pushed, size);
+ return pushed;
+ }
+ });
}
long startTime = System.nanoTime();
long bytes =
@@ -242,7 +234,7 @@ public class VeloxUniffleColumnarShuffleWriter<K, V> extends RssShuffleWriter<K,
closeShuffleWriter();
return super.stop(success);
}
- return Option.empty();
+ return Option.<MapStatus>empty();
}
private void closeShuffleWriter() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org