You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by em...@apache.org on 2020/05/15 04:36:56 UTC
[arrow] branch master updated: ARROW-8169: [Java] Improve the
performance of JDBC adapter by allocating memory proactively
This is an automated email from the ASF dual-hosted git repository.
emkornfield pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 80dc959 ARROW-8169: [Java] Improve the performance of JDBC adapter by allocating memory proactively
80dc959 is described below
commit 80dc95928310c9068b5a1b4bbac9870ac0ba096c
Author: liyafan82 <fa...@foxmail.com>
AuthorDate: Thu May 14 21:36:32 2020 -0700
ARROW-8169: [Java] Improve the performance of JDBC adapter by allocating memory proactively
The current implementation use setSafe methods to dynamically allocate memory if necessary. For fixed width vectors (which are frequently used in JDBC), however, we can allocate memory proactively, since the vector size is known as a configuration parameter. So for fixed width vectors, we can use set methods instead.
This change leads to two benefits:
1. When processing each value, we no longer have to check vector capacity and reallocate memroy if needed. This leads to better performance.
2. If we allow the memory to expand automatically (each time by 2x), the amount of memory usually ends up being more than necessary. By allocating memory by the configuration parameter, we allocate no more, or no less.
Benchmark results show notable performance improvements:
Before:
Benchmark Mode Cnt Score Error Units
JdbcAdapterBenchmarks.consumeBenchmark avgt 5 521.700 ± 4.837 us/op
After:
Benchmark Mode Cnt Score Error Units
JdbcAdapterBenchmarks.consumeBenchmark avgt 5 430.523 ± 9.932 us/op
Closes #6672 from liyafan82/fly_0319_jdbc
Authored-by: liyafan82 <fa...@foxmail.com>
Signed-off-by: Micah Kornfield <em...@gmail.com>
---
.../arrow/adapter/jdbc/ArrowVectorIterator.java | 40 +++++++++++++++++++---
.../org/apache/arrow/adapter/jdbc/JdbcToArrow.java | 3 ++
.../adapter/jdbc/JdbcToArrowConfigBuilder.java | 3 ++
.../arrow/adapter/jdbc/JdbcToArrowUtils.java | 16 +++++++--
.../arrow/adapter/jdbc/consumer/ArrayConsumer.java | 16 +++++++++
.../adapter/jdbc/consumer/BigIntConsumer.java | 8 +++--
.../arrow/adapter/jdbc/consumer/BitConsumer.java | 8 +++--
.../arrow/adapter/jdbc/consumer/DateConsumer.java | 8 +++--
.../adapter/jdbc/consumer/DecimalConsumer.java | 8 +++--
.../adapter/jdbc/consumer/DoubleConsumer.java | 8 +++--
.../arrow/adapter/jdbc/consumer/FloatConsumer.java | 8 +++--
.../arrow/adapter/jdbc/consumer/IntConsumer.java | 8 +++--
.../adapter/jdbc/consumer/SmallIntConsumer.java | 8 +++--
.../arrow/adapter/jdbc/consumer/TimeConsumer.java | 8 +++--
.../adapter/jdbc/consumer/TimestampConsumer.java | 8 +++--
.../adapter/jdbc/consumer/TinyIntConsumer.java | 8 +++--
.../jdbc/h2/JdbcToArrowVectorIteratorTest.java | 12 +++++++
.../arrow/adapter/jdbc/JdbcAdapterBenchmarks.java | 2 +-
18 files changed, 150 insertions(+), 30 deletions(-)
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowVectorIterator.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowVectorIterator.java
index 39d92ff..973bdf1 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowVectorIterator.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowVectorIterator.java
@@ -25,6 +25,8 @@ import java.util.Iterator;
import org.apache.arrow.adapter.jdbc.consumer.CompositeJdbcConsumer;
import org.apache.arrow.adapter.jdbc.consumer.JdbcConsumer;
import org.apache.arrow.util.Preconditions;
+import org.apache.arrow.vector.BaseFixedWidthVector;
+import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.types.pojo.Schema;
@@ -92,12 +94,20 @@ public class ArrowVectorIterator implements Iterator<VectorSchemaRoot>, AutoClos
// consume data
try {
int readRowCount = 0;
- while ((targetBatchSize == JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE || readRowCount < targetBatchSize) &&
- resultSet.next()) {
- compositeConsumer.consume(resultSet);
- readRowCount++;
+ if (targetBatchSize == JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE) {
+ while (resultSet.next()) {
+ ensureCapacity(root, readRowCount + 1);
+ compositeConsumer.consume(resultSet);
+ readRowCount++;
+ }
+ } else {
+ while (readRowCount < targetBatchSize && resultSet.next()) {
+ compositeConsumer.consume(resultSet);
+ readRowCount++;
+ }
}
+
root.setRowCount(readRowCount);
} catch (Exception e) {
compositeConsumer.close();
@@ -109,6 +119,9 @@ public class ArrowVectorIterator implements Iterator<VectorSchemaRoot>, AutoClos
VectorSchemaRoot root = null;
try {
root = VectorSchemaRoot.create(schema, config.getAllocator());
+ if (config.getTargetBatchSize() != JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE) {
+ preAllocate(root, config);
+ }
} catch (Exception e) {
if (root != null) {
root.close();
@@ -118,6 +131,25 @@ public class ArrowVectorIterator implements Iterator<VectorSchemaRoot>, AutoClos
return root;
}
+ static void preAllocate(VectorSchemaRoot root, JdbcToArrowConfig config) {
+ int targetSize = config.getTargetBatchSize();
+ for (ValueVector vector : root.getFieldVectors()) {
+ if (vector instanceof BaseFixedWidthVector) {
+ ((BaseFixedWidthVector) vector).allocateNew(targetSize);
+ }
+ }
+ }
+
+ static void ensureCapacity(VectorSchemaRoot root, int targetCapacity) {
+ for (ValueVector vector : root.getFieldVectors()) {
+ if (vector instanceof BaseFixedWidthVector) {
+ while (vector.getValueCapacity() < targetCapacity) {
+ vector.reAlloc();
+ }
+ }
+ }
+ }
+
// Loads the next schema root or null if no more rows are available.
private void load(VectorSchemaRoot root) throws SQLException {
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java
index d82fb40..80fc81e 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java
@@ -220,6 +220,9 @@ public class JdbcToArrow {
VectorSchemaRoot root = VectorSchemaRoot.create(
JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData(), config), config.getAllocator());
+ if (config.getTargetBatchSize() != JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE) {
+ ArrowVectorIterator.preAllocate(root, config);
+ }
JdbcToArrowUtils.jdbcToArrowVectors(resultSet, root, config);
return root;
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java
index 3a4758d..83a3baa 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java
@@ -17,6 +17,8 @@
package org.apache.arrow.adapter.jdbc;
+import static org.apache.arrow.adapter.jdbc.JdbcToArrowConfig.DEFAULT_TARGET_BATCH_SIZE;
+
import java.util.Calendar;
import java.util.Map;
@@ -71,6 +73,7 @@ public class JdbcToArrowConfigBuilder {
this.allocator = allocator;
this.calendar = calendar;
this.includeMetadata = false;
+ this.targetBatchSize = DEFAULT_TARGET_BATCH_SIZE;
}
/**
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java
index f7bf324..081f179 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java
@@ -355,16 +355,26 @@ public class JdbcToArrowUtils {
try {
compositeConsumer = new CompositeJdbcConsumer(consumers);
int readRowCount = 0;
- while (rs.next()) {
- compositeConsumer.consume(rs);
- readRowCount++;
+ if (config.getTargetBatchSize() == JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE) {
+ while (rs.next()) {
+ ArrowVectorIterator.ensureCapacity(root, readRowCount + 1);
+ compositeConsumer.consume(rs);
+ readRowCount++;
+ }
+ } else {
+ while (rs.next() && readRowCount < config.getTargetBatchSize()) {
+ compositeConsumer.consume(rs);
+ readRowCount++;
+ }
}
+
root.setRowCount(readRowCount);
} catch (Exception e) {
// error occurs and clean up resources.
if (compositeConsumer != null) {
compositeConsumer.close();
}
+ throw e;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ArrayConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ArrayConsumer.java
index acedbd8..ed12f81 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ArrayConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ArrayConsumer.java
@@ -22,6 +22,7 @@ import java.sql.Array;
import java.sql.ResultSet;
import java.sql.SQLException;
+import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.complex.ListVector;
/**
@@ -44,12 +45,17 @@ public abstract class ArrayConsumer extends BaseConsumer<ListVector> {
protected final JdbcConsumer delegate;
+ private final ValueVector innerVector;
+
+ protected int innerVectorIndex = 0;
+
/**
* Instantiate a ArrayConsumer.
*/
public ArrayConsumer(ListVector vector, JdbcConsumer delegate, int index) {
super(vector, index);
this.delegate = delegate;
+ this.innerVector = vector.getDataVector();
}
@Override
@@ -58,6 +64,12 @@ public abstract class ArrayConsumer extends BaseConsumer<ListVector> {
this.delegate.close();
}
+ void ensureInnerVectorCapacity(int targetCapacity) {
+ while (innerVector.getValueCapacity() < targetCapacity) {
+ innerVector.reAlloc();
+ }
+ }
+
/**
* Nullable consumer for {@link ListVector}.
*/
@@ -78,11 +90,13 @@ public abstract class ArrayConsumer extends BaseConsumer<ListVector> {
int count = 0;
try (ResultSet rs = array.getResultSet()) {
while (rs.next()) {
+ ensureInnerVectorCapacity(innerVectorIndex + count + 1);
delegate.consume(rs);
count++;
}
}
vector.endValue(currentIndex, count);
+ innerVectorIndex += count;
}
currentIndex++;
}
@@ -107,11 +121,13 @@ public abstract class ArrayConsumer extends BaseConsumer<ListVector> {
int count = 0;
try (ResultSet rs = array.getResultSet()) {
while (rs.next()) {
+ ensureInnerVectorCapacity(innerVectorIndex + count + 1);
delegate.consume(rs);
count++;
}
}
vector.endValue(currentIndex, count);
+ innerVectorIndex += count;
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BigIntConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BigIntConsumer.java
index 09d26b8..19c8efa 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BigIntConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BigIntConsumer.java
@@ -55,7 +55,9 @@ public class BigIntConsumer {
public void consume(ResultSet resultSet) throws SQLException {
long value = resultSet.getLong(columnIndexInResultSet);
if (!resultSet.wasNull()) {
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
}
currentIndex++;
}
@@ -76,7 +78,9 @@ public class BigIntConsumer {
@Override
public void consume(ResultSet resultSet) throws SQLException {
long value = resultSet.getLong(columnIndexInResultSet);
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BitConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BitConsumer.java
index 27c453d..d2d94d0 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BitConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BitConsumer.java
@@ -55,7 +55,9 @@ public class BitConsumer {
public void consume(ResultSet resultSet) throws SQLException {
boolean value = resultSet.getBoolean(columnIndexInResultSet);
if (!resultSet.wasNull()) {
- vector.setSafe(currentIndex, value ? 1 : 0);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value ? 1 : 0);
}
currentIndex++;
}
@@ -76,7 +78,9 @@ public class BitConsumer {
@Override
public void consume(ResultSet resultSet) throws SQLException {
boolean value = resultSet.getBoolean(columnIndexInResultSet);
- vector.setSafe(currentIndex, value ? 1 : 0);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value ? 1 : 0);
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java
index bb9ab1c..c431bc7 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java
@@ -89,7 +89,9 @@ public class DateConsumer {
if (day < 0 || day > MAX_DAY) {
throw new IllegalArgumentException("Day overflow: " + day);
}
- vector.setSafe(currentIndex, day);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, day);
}
currentIndex++;
}
@@ -125,7 +127,9 @@ public class DateConsumer {
if (day < 0 || day > MAX_DAY) {
throw new IllegalArgumentException("Day overflow: " + day);
}
- vector.setSafe(currentIndex, day);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, day);
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DecimalConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DecimalConsumer.java
index 080dd41..4498fde 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DecimalConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DecimalConsumer.java
@@ -56,7 +56,9 @@ public class DecimalConsumer {
public void consume(ResultSet resultSet) throws SQLException {
BigDecimal value = resultSet.getBigDecimal(columnIndexInResultSet);
if (!resultSet.wasNull()) {
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
}
currentIndex++;
}
@@ -77,7 +79,9 @@ public class DecimalConsumer {
@Override
public void consume(ResultSet resultSet) throws SQLException {
BigDecimal value = resultSet.getBigDecimal(columnIndexInResultSet);
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DoubleConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DoubleConsumer.java
index d884be8..e3db95d 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DoubleConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DoubleConsumer.java
@@ -55,7 +55,9 @@ public class DoubleConsumer {
public void consume(ResultSet resultSet) throws SQLException {
double value = resultSet.getDouble(columnIndexInResultSet);
if (!resultSet.wasNull()) {
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
}
currentIndex++;
}
@@ -76,7 +78,9 @@ public class DoubleConsumer {
@Override
public void consume(ResultSet resultSet) throws SQLException {
double value = resultSet.getDouble(columnIndexInResultSet);
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/FloatConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/FloatConsumer.java
index 9dd4536..830348f 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/FloatConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/FloatConsumer.java
@@ -55,7 +55,9 @@ public class FloatConsumer {
public void consume(ResultSet resultSet) throws SQLException {
float value = resultSet.getFloat(columnIndexInResultSet);
if (!resultSet.wasNull()) {
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
}
currentIndex++;
}
@@ -76,7 +78,9 @@ public class FloatConsumer {
@Override
public void consume(ResultSet resultSet) throws SQLException {
float value = resultSet.getFloat(columnIndexInResultSet);
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/IntConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/IntConsumer.java
index 2bf050c..4e537d6 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/IntConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/IntConsumer.java
@@ -55,7 +55,9 @@ public class IntConsumer {
public void consume(ResultSet resultSet) throws SQLException {
int value = resultSet.getInt(columnIndexInResultSet);
if (!resultSet.wasNull()) {
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
}
currentIndex++;
}
@@ -76,7 +78,9 @@ public class IntConsumer {
@Override
public void consume(ResultSet resultSet) throws SQLException {
int value = resultSet.getInt(columnIndexInResultSet);
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/SmallIntConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/SmallIntConsumer.java
index 6ec773d..2edb360 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/SmallIntConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/SmallIntConsumer.java
@@ -55,7 +55,9 @@ public class SmallIntConsumer {
public void consume(ResultSet resultSet) throws SQLException {
short value = resultSet.getShort(columnIndexInResultSet);
if (!resultSet.wasNull()) {
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
}
currentIndex++;
}
@@ -76,7 +78,9 @@ public class SmallIntConsumer {
@Override
public void consume(ResultSet resultSet) throws SQLException {
short value = resultSet.getShort(columnIndexInResultSet);
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimeConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimeConsumer.java
index 3edfd03..4fa15ad 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimeConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimeConsumer.java
@@ -69,7 +69,9 @@ public abstract class TimeConsumer {
Time time = calendar == null ? resultSet.getTime(columnIndexInResultSet) :
resultSet.getTime(columnIndexInResultSet, calendar);
if (!resultSet.wasNull()) {
- vector.setSafe(currentIndex, (int) time.getTime());
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, (int) time.getTime());
}
currentIndex++;
}
@@ -101,7 +103,9 @@ public abstract class TimeConsumer {
public void consume(ResultSet resultSet) throws SQLException {
Time time = calendar == null ? resultSet.getTime(columnIndexInResultSet) :
resultSet.getTime(columnIndexInResultSet, calendar);
- vector.setSafe(currentIndex, (int) time.getTime());
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, (int) time.getTime());
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampConsumer.java
index 3f79da4..7635da1 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampConsumer.java
@@ -69,7 +69,9 @@ public abstract class TimestampConsumer {
Timestamp timestamp = calendar == null ? resultSet.getTimestamp(columnIndexInResultSet) :
resultSet.getTimestamp(columnIndexInResultSet, calendar);
if (!resultSet.wasNull()) {
- vector.setSafe(currentIndex, timestamp.getTime());
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, timestamp.getTime());
}
currentIndex++;
}
@@ -101,7 +103,9 @@ public abstract class TimestampConsumer {
public void consume(ResultSet resultSet) throws SQLException {
Timestamp timestamp = calendar == null ? resultSet.getTimestamp(columnIndexInResultSet) :
resultSet.getTimestamp(columnIndexInResultSet, calendar);
- vector.setSafe(currentIndex, timestamp.getTime());
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, timestamp.getTime());
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TinyIntConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TinyIntConsumer.java
index 858116f..40cf087 100644
--- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TinyIntConsumer.java
+++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TinyIntConsumer.java
@@ -55,7 +55,9 @@ public abstract class TinyIntConsumer {
public void consume(ResultSet resultSet) throws SQLException {
byte value = resultSet.getByte(columnIndexInResultSet);
if (!resultSet.wasNull()) {
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
}
currentIndex++;
}
@@ -76,7 +78,9 @@ public abstract class TinyIntConsumer {
@Override
public void consume(ResultSet resultSet) throws SQLException {
byte value = resultSet.getByte(columnIndexInResultSet);
- vector.setSafe(currentIndex, value);
+ // for fixed width vectors, we have allocated enough memory proactively,
+ // so there is no need to call the setSafe method here.
+ vector.set(currentIndex, value);
currentIndex++;
}
}
diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java
index fa75c1c..fa947d8 100644
--- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java
+++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java
@@ -80,6 +80,18 @@ public class JdbcToArrowVectorIteratorTest extends JdbcToArrowTest {
validate(iterator);
}
+ @Test
+ public void testJdbcToArrowValuesNoLimit() throws SQLException, IOException {
+
+ JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE),
+ Calendar.getInstance()).setTargetBatchSize(JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE).build();
+
+ ArrowVectorIterator iterator =
+ JdbcToArrow.sqlToArrowVectorIterator(conn.createStatement().executeQuery(table.getQuery()), config);
+
+ validate(iterator);
+ }
+
private void validate(ArrowVectorIterator iterator) throws SQLException, IOException {
List<BigIntVector> bigIntVectors = new ArrayList<>();
diff --git a/java/performance/src/test/java/org/apache/arrow/adapter/jdbc/JdbcAdapterBenchmarks.java b/java/performance/src/test/java/org/apache/arrow/adapter/jdbc/JdbcAdapterBenchmarks.java
index c6ef1a2..3822088 100644
--- a/java/performance/src/test/java/org/apache/arrow/adapter/jdbc/JdbcAdapterBenchmarks.java
+++ b/java/performance/src/test/java/org/apache/arrow/adapter/jdbc/JdbcAdapterBenchmarks.java
@@ -255,7 +255,7 @@ public class JdbcAdapterBenchmarks {
@Setup(Level.Trial)
public void prepareState() throws Exception {
allocator = new RootAllocator(Integer.MAX_VALUE);
- config = new JdbcToArrowConfigBuilder().setAllocator(allocator).setTargetBatchSize(1024).build();
+ config = new JdbcToArrowConfigBuilder().setAllocator(allocator).setTargetBatchSize(VALUE_COUNT).build();
Class.forName(DRIVER);
conn = DriverManager.getConnection(URL);