You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pr...@apache.org on 2017/12/21 05:19:33 UTC
[06/15] drill git commit: DRILL-5657: Size-aware vector writer
structure
http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java b/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java
deleted file mode 100644
index 84961b1..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java
+++ /dev/null
@@ -1,487 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.vector;
-
-import static org.junit.Assert.*;
-
-import org.apache.drill.categories.VectorTest;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.vector.IntVector;
-import org.apache.drill.exec.vector.NullableIntVector;
-import org.apache.drill.exec.vector.NullableVarCharVector;
-import org.apache.drill.exec.vector.RepeatedIntVector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VarCharVector;
-import org.apache.drill.exec.vector.VectorOverflowException;
-import org.apache.drill.test.DrillTest;
-import org.apache.drill.test.OperatorFixture;
-import org.bouncycastle.util.Arrays;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import io.netty.buffer.DrillBuf;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test the setScalar() methods in the various generated vector
- * classes. Rather than test all 100+ vectors, we sample a few and
- * rely on the fact that code is generated from a common template.
- */
-
-@Category(VectorTest.class)
-public class TestVectorLimits extends DrillTest {
-
- public static OperatorFixture fixture;
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- fixture = OperatorFixture.builder().build();
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- fixture.close();
- }
-
- /**
- * Test a vector directly using the vector mutator to ensure
- * that the <tt>setScalar</tt> method works for the maximum
- * row count.
- * <p>
- * This test is a proxy for all the other fixed types, since all
- * share the same code template.
- */
-
- @Test
- public void testFixedVector() {
-
- // Create a non-nullable int vector: a typical fixed-size vector
-
- @SuppressWarnings("resource")
- IntVector vector = new IntVector(makeField(MinorType.INT, DataMode.REQUIRED), fixture.allocator() );
-
- // Sanity test of generated constants.
-
- assertTrue( IntVector.MAX_SCALAR_COUNT <= ValueVector.MAX_ROW_COUNT );
- assertEquals( 4, IntVector.VALUE_WIDTH );
- assertTrue( IntVector.NET_MAX_SCALAR_SIZE <= ValueVector.MAX_BUFFER_SIZE );
-
- // Allocate a default size, small vector. Forces test of
- // the auto-grow (setSafe()) aspect of setScalar().
-
- vector.allocateNew( );
-
- // Write to the vector until it complains. At that point,
- // we should have written up to the static fixed value count
- // (which is computed to stay below the capacity limit.)
-
- IntVector.Mutator mutator = vector.getMutator();
- for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) {
- try {
- mutator.setScalar(i, i);
- } catch (VectorOverflowException e) {
- assertEquals(IntVector.MAX_SCALAR_COUNT, i);
- break;
- }
- }
-
- // The vector should be below the allocation limit. Since this
- // is an int vector, in practice the size will be far below
- // the overall limit (if the limit stays at 16 MB.) But, it should
- // be at the type-specific limit since we filled up the vector.
-
- assertEquals(IntVector.NET_MAX_SCALAR_SIZE, vector.getBuffer().getActualMemoryConsumed());
- vector.close();
- }
-
- @Test
- public void testNullableFixedVector() {
-
- @SuppressWarnings("resource")
- NullableIntVector vector = new NullableIntVector(makeField(MinorType.INT, DataMode.OPTIONAL), fixture.allocator() );
- vector.allocateNew( );
-
- NullableIntVector.Mutator mutator = vector.getMutator();
- for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) {
- try {
- mutator.setScalar(i, i);
- } catch (VectorOverflowException e) {
- assertEquals(IntVector.MAX_SCALAR_COUNT, i);
- break;
- }
- }
-
- vector.close();
- }
-
- /**
- * Repeated fixed vector. Using an int vector, each column array can hold
- * 256 / 4 = 64 values. We write only 10. The vector becomes full when we
- * exceed 64K items.
- */
-
- @Test
- public void testRepeatedFixedVectorCountLimit() {
-
- @SuppressWarnings("resource")
- RepeatedIntVector vector = new RepeatedIntVector(makeField(MinorType.INT, DataMode.REPEATED), fixture.allocator() );
- vector.allocateNew( );
-
- RepeatedIntVector.Mutator mutator = vector.getMutator();
- top:
- for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) {
- if (! mutator.startNewValueBounded(i)) {
- assertEquals(ValueVector.MAX_ROW_COUNT, i);
- // Continue, let's check the addBounded method also
- }
- for (int j = 0; j < 10; j++) {
- try {
- mutator.addEntry(i, i * 100 + j);
- } catch (VectorOverflowException e) {
- assertEquals(ValueVector.MAX_ROW_COUNT, i);
- mutator.setValueCount(i);
- break top;
- }
- }
- }
-
- vector.close();
- }
-
- /**
- * Repeated fixed vector. Using an int vector, each column array can hold
- * 256 / 4 = 64 values. We write 100. The vector becomes full when we
- * exceed the 16 MB size limit.
- */
-
- @Test
- public void testRepeatedFixedVectorBufferLimit() {
-
- @SuppressWarnings("resource")
- RepeatedIntVector vector = new RepeatedIntVector(makeField(MinorType.INT, DataMode.REPEATED), fixture.allocator() );
- vector.allocateNew( );
-
- RepeatedIntVector.Mutator mutator = vector.getMutator();
- top:
- for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) {
- // We'll never hit the value count limit
- assertTrue(mutator.startNewValueBounded(i));
- for (int j = 0; j < 100; j++) {
- try {
- mutator.addEntry(i, i * 100 + j);
- } catch (VectorOverflowException e) {
- // We should have hit the buffer limit before the value limit.
- assertTrue(i < ValueVector.MAX_ROW_COUNT);
- mutator.setValueCount(i);
- break top;
- }
- }
- }
-
- vector.close();
- }
-
- // To be replaced by a test method in a separate commit.
-
- public static MaterializedField makeField(MinorType dataType, DataMode mode) {
- MajorType type = MajorType.newBuilder()
- .setMinorType(dataType)
- .setMode(mode)
- .build();
-
- return MaterializedField.create("foo", type);
- }
-
- /**
- * Baseline test for a variable-width vector using <tt>setSafe</tt> and
- * loading the vector up to the maximum size. Doing so will cause the vector
- * to have a buffer that exceeds the maximum size, demonstrating the
- * need for <tt>setScalar()</tt>.
- */
-
- @Test
- public void variableVectorBaseline() {
-
- // Create a non-nullable VarChar vector: a typical variable-size vector
-
- @SuppressWarnings("resource")
- VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() );
- vector.allocateNew( );
-
- // A 16 MB value can hold 64K values of up to 256 bytes each.
- // To force a size overflow, write values much larger.
- // Write the maximum number of values which will silently
- // allow the vector to grow beyond the critical size of 16 MB.
- // Doing this in production would lead to memory fragmentation.
- // So, this is what the setScalar() method assures we don't do.
-
- byte dummyValue[] = new byte[512];
- Arrays.fill(dummyValue, (byte) 'X');
- VarCharVector.Mutator mutator = vector.getMutator();
- for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) {
- mutator.setSafe(i, dummyValue, 0, dummyValue.length);
- }
-
- // The vector should be above the allocation limit.
- // This is why code must migrate to the setScalar() call
- // away from the setSafe() call.
-
- assertTrue(ValueVector.MAX_BUFFER_SIZE < vector.getBuffer().getActualMemoryConsumed());
- vector.close();
- }
-
- /**
- * Test a vector directly using the vector mutator to ensure
- * that the <tt>setScalar</tt> method works for the maximum
- * vector size.
- */
-
- @Test
- public void testWideVariableVector() {
-
- @SuppressWarnings("resource")
- VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() );
- vector.allocateNew( );
-
- // A 16 MB value can hold 64K values of up to 256 bytes each.
- // To force a size overflow, write values much larger.
- // Write to the vector until it complains. At that point,
- // we should have written up to the maximum buffer size.
-
- byte dummyValue[] = makeVarCharValue(512);
- VarCharVector.Mutator mutator = vector.getMutator();
- int count = 0;
- for ( ; count < 2 * ValueVector.MAX_ROW_COUNT; count++) {
- try {
- mutator.setScalar(count, dummyValue, 0, dummyValue.length);
- } catch (VectorOverflowException e) {
- break;
- }
- }
-
- // The vector should be at the allocation limit. If it wasn't, we
- // should have grown it to hold more data. The value count will
- // be below the maximum.
-
- mutator.setValueCount(count);
- assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getBuffer().getActualMemoryConsumed());
- assertTrue(count < ValueVector.MAX_ROW_COUNT);
- vector.close();
- }
-
- private byte[] makeVarCharValue(int n) {
- byte dummyValue[] = new byte[n];
- Arrays.fill(dummyValue, (byte) 'X');
- return dummyValue;
- }
-
- @Test
- public void testNullableWideVariableVector() {
-
- @SuppressWarnings("resource")
- NullableVarCharVector vector = new NullableVarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() );
- vector.allocateNew( );
-
- byte dummyValue[] = makeVarCharValue(512);
- NullableVarCharVector.Mutator mutator = vector.getMutator();
- int count = 0;
- for ( ; count < 2 * ValueVector.MAX_ROW_COUNT; count++) {
- try {
- mutator.setScalar(count, dummyValue, 0, dummyValue.length);
- } catch (VectorOverflowException e) {
- break;
- }
- }
-
- mutator.setValueCount(count);
- assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getValuesVector().getBuffer().getActualMemoryConsumed());
- assertTrue(count < ValueVector.MAX_ROW_COUNT);
- vector.close();
- }
-
- /**
- * Test a vector directly using the vector mutator to ensure
- * that the <tt>setScalar</tt> method works for the maximum
- * value count.
- */
-
- @Test
- public void testNarrowVariableVector() {
-
- @SuppressWarnings("resource")
- VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() );
- vector.allocateNew( );
-
- // Write small values that fit into 16 MB. We should stop writing
- // when we reach the value count limit.
-
- byte dummyValue[] = makeVarCharValue(254);
- VarCharVector.Mutator mutator = vector.getMutator();
- int count = 0;
- for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) {
- try {
- mutator.setScalar(count, dummyValue, 0, dummyValue.length);
- } catch (VectorOverflowException e) {
- break;
- }
- }
-
- // Buffer size should be at or below the maximum, with count
- // at the maximum.
-
- mutator.setValueCount(count);
- assertTrue(vector.getBuffer().getActualMemoryConsumed() <= ValueVector.MAX_BUFFER_SIZE);
- assertEquals(ValueVector.MAX_ROW_COUNT, count);
- vector.close();
- }
-
- /**
- * Test a vector directly using the vector mutator to ensure
- * that the <tt>setScalar</tt> method works for the maximum
- * value count. Uses a DrillBuf as input.
- */
-
- @Test
- public void testDirectVariableVector() {
-
- @SuppressWarnings("resource")
- VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() );
- vector.allocateNew( );
-
- // Repeat the big-value test, but with data coming from a DrillBuf
- // (direct memory) rather than a heap array.
-
- @SuppressWarnings("resource")
- DrillBuf drillBuf = makeVarCharValueDirect(260);
- VarCharVector.Mutator mutator = vector.getMutator();
- int count = 0;
- for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) {
- try {
- mutator.setScalar(count, drillBuf, 0, 260);
- } catch (VectorOverflowException e) {
- break;
- }
- }
- drillBuf.close();
-
- // Again, vector should be at the size limit, count below the
- // value limit.
-
- mutator.setValueCount(count);
- assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getBuffer().getActualMemoryConsumed());
- assertTrue(count < ValueVector.MAX_ROW_COUNT);
- vector.close();
- }
-
- private DrillBuf makeVarCharValueDirect(int n) {
- byte dummyValue[] = makeVarCharValue(n);
- DrillBuf drillBuf = fixture.allocator().buffer(dummyValue.length);
- drillBuf.setBytes(0, dummyValue);
- return drillBuf;
- }
-
- @Test
- public void testDirectNullableVariableVector() {
-
- @SuppressWarnings("resource")
- NullableVarCharVector vector = new NullableVarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() );
- vector.allocateNew( );
-
- @SuppressWarnings("resource")
- DrillBuf drillBuf = makeVarCharValueDirect(260);
- NullableVarCharVector.Mutator mutator = vector.getMutator();
- int count = 0;
- for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) {
- try {
- mutator.setScalar(count, drillBuf, 0, 260);
- } catch (VectorOverflowException e) {
- break;
- }
- }
- drillBuf.close();
-
- mutator.setValueCount(count);
- assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getValuesVector().getBuffer().getActualMemoryConsumed());
- assertTrue(count < ValueVector.MAX_ROW_COUNT);
- vector.close();
- }
-
- public static void main(String args[]) {
- try {
- setUpBeforeClass();
- new TestVectorLimits().performanceTest();
- tearDownAfterClass();
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
-
- private void performanceTest() {
- @SuppressWarnings("resource")
- VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() );
- byte value[] = makeVarCharValue(1);
- int warmCount = 100;
- timeSetSafe(vector, value, warmCount);
- runSetBounded(vector, value, warmCount);
- int runCount = 1000;
- timeSetSafe(vector, value, runCount);
- runSetBounded(vector, value, runCount);
- timeSetSafe(vector, value, runCount);
- vector.close();
- }
-
- private void timeSetSafe(VarCharVector vector, byte[] value, int iterCount) {
- long start = System.currentTimeMillis();
- for (int i = 0; i < iterCount; i++) {
- vector.clear();
- vector.allocateNew( );
-
- VarCharVector.Mutator mutator = vector.getMutator();
- for (int j = 0; j < ValueVector.MAX_ROW_COUNT; j++) {
- mutator.setSafe(j, value, 0, value.length);
- }
- }
- long elapsed = System.currentTimeMillis() - start;
- System.out.println( iterCount + " runs of setSafe: " + elapsed + " ms." );
- }
-
- private void runSetBounded(VarCharVector vector, byte[] value, int iterCount) {
- long start = System.currentTimeMillis();
- for (int i = 0; i < iterCount; i++) {
- vector.clear();
- vector.allocateNew( );
-
- VarCharVector.Mutator mutator = vector.getMutator();
- int posn = 0;
- for (;;) {
- try {
- mutator.setScalar(posn++, value, 0, value.length);
- } catch (VectorOverflowException e) {
- break;
- }
- }
- }
- long elapsed = System.currentTimeMillis() - start;
- System.out.println( iterCount + " runs of setScalar: " + elapsed + " ms." );
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/jdbc-all/pom.xml
----------------------------------------------------------------------
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index 82910af..9db9c58 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -509,7 +509,7 @@
This is likely due to you adding new dependencies to a java-exec and not updating the excludes in this module. This is important as it minimizes the size of the dependency of Drill application users.
</message>
- <maxsize>30000000</maxsize>
+ <maxsize>32000000</maxsize>
<minsize>15000000</minsize>
<files>
<file>${project.build.directory}/drill-jdbc-all-${project.version}.jar</file>
http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
index 9019507..eda189e 100644
--- a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
+++ b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
@@ -52,7 +52,6 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
private final int offset;
private final BufferLedger ledger;
private final BufferManager bufManager;
-// private final ByteBufAllocator alloc;
private final boolean isEmpty;
private volatile int length;
private final HistoricalLog historicalLog = BaseAllocator.DEBUG ?
@@ -72,7 +71,6 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
this.udle = byteBuf;
this.isEmpty = isEmpty;
this.bufManager = manager;
-// this.alloc = alloc;
this.addr = byteBuf.memoryAddress() + offset;
this.ledger = ledger;
this.length = length;
@@ -106,6 +104,8 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
}
}
+ public long addr() { return addr; }
+
private long addr(int index) {
return addr + index;
}
@@ -882,4 +882,70 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
}
}
+ // The "unsafe" methods are for use ONLY by code that does its own
+ // bounds checking. They are called "unsafe" for a reason: they will crash
+ // the JVM if values are addressed out of bounds.
+
+ /**
+ * Write an integer to the buffer at the given byte index, without
+ * bounds checks.
+ *
+ * @param offset byte (not int) offset of the location to write
+ * @param value the value to write
+ */
+
+ public void unsafePutInt(int offset, int value) {
+ PlatformDependent.putInt(addr + offset, value);
+ }
+
+ /**
+ * Write a long to the buffer at the given byte index, without
+ * bounds checks.
+ *
+ * @param index byte (not long) offset of the location to write
+ * @param value the value to write
+ */
+
+ public void unsafePutLong(int index, long value) {
+ PlatformDependent.putLong(addr + index, value);
+ }
+
+ /**
+ * Write a short to the buffer at the given byte index, without
+ * bounds checks.
+ *
+ * @param offset byte (not short) offset of the location to write
+ * @param value the value to write
+ */
+
+ public void unsafePutShort(int offset, short value) {
+ PlatformDependent.putShort(addr + offset, value);
+ }
+
+ /**
+ * Write a byte to the buffer at the given byte index, without
+ * bounds checks.
+ *
+ * @param offset byte offset of the location to write
+ * @param value the value to write
+ */
+
+ public void unsafePutByte(int offset, byte value) {
+ PlatformDependent.putByte(addr + offset, value);
+ }
+
+ /**
+ * Copy a buffer of heap data to the buffer memory.
+ *
+ * @param srce source byte buffer
+ * @param srcOffset offset within the byte buffer of the start of data
+ * @param destOffset byte offset into this buffer to which to write the
+ * data
+ * @param length length of the data, which must be within the
+ * bounds of this buffer
+ */
+
+ public void unsafeCopyMemory(byte[] srce, int srcOffset, int destOffset, int length) {
+ PlatformDependent.copyMemory(srce, srcOffset, addr + destOffset, length);
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java b/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
index 5358ca2..1e70216 100644
--- a/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ b/exec/memory/base/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -32,14 +32,16 @@ import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
/**
- * The base allocator that we use for all of Drill's memory management. Returns UnsafeDirectLittleEndian buffers.
+ * The base allocator that we use for all of Drill's memory management. Returns
+ * UnsafeDirectLittleEndian buffers.
*/
+
public class PooledByteBufAllocatorL {
- private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory.getLogger("drill.allocator");
+ private static final org.slf4j.Logger memoryLogger = org.slf4j.LoggerFactory
+ .getLogger("drill.allocator");
private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60;
-
public static final String METRIC_PREFIX = "drill.allocator.";
private final MetricRegistry registry;
@@ -54,7 +56,8 @@ public class PooledByteBufAllocatorL {
public PooledByteBufAllocatorL(MetricRegistry registry) {
this.registry = registry;
allocator = new InnerAllocator();
- empty = new UnsafeDirectLittleEndian(new DuplicatedByteBuf(Unpooled.EMPTY_BUFFER));
+ empty = new UnsafeDirectLittleEndian(
+ new DuplicatedByteBuf(Unpooled.EMPTY_BUFFER));
}
public UnsafeDirectLittleEndian allocate(int size) {
@@ -63,7 +66,6 @@ public class PooledByteBufAllocatorL {
} catch (OutOfMemoryError e) {
throw new OutOfMemoryException("Failure allocating buffer.", e);
}
-
}
public int getChunkSize() {
@@ -72,13 +74,13 @@ public class PooledByteBufAllocatorL {
private class InnerAllocator extends PooledByteBufAllocator {
-
private final PoolArena<ByteBuffer>[] directArenas;
private final MemoryStatusThread statusThread;
private final Histogram largeBuffersHist;
private final Histogram normalBuffersHist;
private final int chunkSize;
+ @SuppressWarnings("unchecked")
public InnerAllocator() {
super(true);
@@ -87,7 +89,9 @@ public class PooledByteBufAllocatorL {
f.setAccessible(true);
this.directArenas = (PoolArena<ByteBuffer>[]) f.get(this);
} catch (Exception e) {
- throw new RuntimeException("Failure while initializing allocator. Unable to retrieve direct arenas field.", e);
+ throw new RuntimeException(
+ "Failure while initializing allocator. Unable to retrieve direct arenas field.",
+ e);
}
this.chunkSize = directArenas[0].chunkSize;
@@ -130,17 +134,14 @@ public class PooledByteBufAllocatorL {
largeBuffersHist = registry.histogram(METRIC_PREFIX + "huge.hist");
normalBuffersHist = registry.histogram(METRIC_PREFIX + "normal.hist");
-
}
-
private synchronized void removeOldMetrics() {
registry.removeMatching(new MetricFilter() {
@Override
public boolean matches(String name, Metric metric) {
return name.startsWith("drill.allocator.");
}
-
});
}
@@ -152,17 +153,21 @@ public class PooledByteBufAllocatorL {
if (initialCapacity > directArena.chunkSize) {
// This is beyond chunk size so we'll allocate separately.
- ByteBuf buf = UnpooledByteBufAllocator.DEFAULT.directBuffer(initialCapacity, maxCapacity);
+ ByteBuf buf = UnpooledByteBufAllocator.DEFAULT
+ .directBuffer(initialCapacity, maxCapacity);
hugeBufferCount.incrementAndGet();
hugeBufferSize.addAndGet(buf.capacity());
largeBuffersHist.update(buf.capacity());
- // logger.debug("Allocating huge buffer of size {}", initialCapacity, new Exception());
- return new UnsafeDirectLittleEndian(new LargeBuffer(buf, hugeBufferSize, hugeBufferCount));
+ // logger.debug("Allocating huge buffer of size {}", initialCapacity,
+ // new Exception());
+ return new UnsafeDirectLittleEndian(
+ new LargeBuffer(buf, hugeBufferSize, hugeBufferCount));
} else {
// within chunk, use arena.
- ByteBuf buf = directArena.allocate(cache, initialCapacity, maxCapacity);
+ ByteBuf buf = directArena.allocate(cache, initialCapacity,
+ maxCapacity);
if (!(buf instanceof PooledUnsafeDirectByteBuf)) {
fail();
}
@@ -173,10 +178,9 @@ public class PooledByteBufAllocatorL {
normalBufferCount.incrementAndGet();
}
- return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf, normalBufferCount,
- normalBufferSize);
+ return new UnsafeDirectLittleEndian((PooledUnsafeDirectByteBuf) buf,
+ normalBufferCount, normalBufferSize);
}
-
} else {
throw fail();
}
@@ -184,10 +188,12 @@ public class PooledByteBufAllocatorL {
private UnsupportedOperationException fail() {
return new UnsupportedOperationException(
- "Drill requries that the JVM used supports access sun.misc.Unsafe. This platform didn't provide that functionality.");
+ "Drill requires that the JVM used supports access sun.misc.Unsafe. This platform doesn't provide that functionality.");
}
- public UnsafeDirectLittleEndian directBuffer(int initialCapacity, int maxCapacity) {
+ @Override
+ public UnsafeDirectLittleEndian directBuffer(int initialCapacity,
+ int maxCapacity) {
if (initialCapacity == 0 && maxCapacity == 0) {
newDirectBuffer(initialCapacity, maxCapacity);
}
@@ -197,13 +203,14 @@ public class PooledByteBufAllocatorL {
@Override
public ByteBuf heapBuffer(int initialCapacity, int maxCapacity) {
- throw new UnsupportedOperationException("Drill doesn't support using heap buffers.");
+ throw new UnsupportedOperationException(
+ "Drill doesn't support using heap buffers.");
}
-
private void validate(int initialCapacity, int maxCapacity) {
if (initialCapacity < 0) {
- throw new IllegalArgumentException("initialCapacity: " + initialCapacity + " (expectd: 0+)");
+ throw new IllegalArgumentException(
+ "initialCapacity: " + initialCapacity + " (expected: 0+)");
}
if (initialCapacity > maxCapacity) {
throw new IllegalArgumentException(String.format(
@@ -223,18 +230,18 @@ public class PooledByteBufAllocatorL {
@Override
public void run() {
while (true) {
- memoryLogger.trace("Memory Usage: \n{}", PooledByteBufAllocatorL.this.toString());
+ memoryLogger.trace("Memory Usage: \n{}",
+ PooledByteBufAllocatorL.this.toString());
try {
Thread.sleep(MEMORY_LOGGER_FREQUENCY_SECONDS * 1000);
} catch (InterruptedException e) {
return;
}
-
}
}
-
}
+ @Override
public String toString() {
StringBuilder buf = new StringBuilder();
buf.append(directArenas.length);
@@ -257,8 +264,6 @@ public class PooledByteBufAllocatorL {
buf.append(" bytes.");
return buf.toString();
}
-
-
}
public static final boolean ASSERT_ENABLED;
@@ -268,5 +273,4 @@ public class PooledByteBufAllocatorL {
assert isAssertEnabled = true;
ASSERT_ENABLED = isAssertEnabled;
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/memory/base/src/main/java/org/apache/drill/exec/memory/AllocationManager.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/AllocationManager.java b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/AllocationManager.java
index 833a604..3b5967f 100644
--- a/exec/memory/base/src/main/java/org/apache/drill/exec/memory/AllocationManager.java
+++ b/exec/memory/base/src/main/java/org/apache/drill/exec/memory/AllocationManager.java
@@ -38,23 +38,25 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
/**
- * Manages the relationship between one or more allocators and a particular UDLE. Ensures that one allocator owns the
- * memory that multiple allocators may be referencing. Manages a BufferLedger between each of its associated allocators.
- * This class is also responsible for managing when memory is allocated and returned to the Netty-based
- * PooledByteBufAllocatorL.
+ * Manages the relationship between one or more allocators and a particular
+ * UDLE. Ensures that one allocator owns the memory that multiple allocators may
+ * be referencing. Manages a BufferLedger between each of its associated
+ * allocators. This class is also responsible for managing when memory is
+ * allocated and returned to the Netty-based PooledByteBufAllocatorL.
*
- * The only reason that this isn't package private is we're forced to put DrillBuf in Netty's package which need access
- * to these objects or methods.
+ * The only reason that this isn't package private is we're forced to put
+ * DrillBuf in Netty's package which need access to these objects or methods.
*
- * Threading: AllocationManager manages thread-safety internally. Operations within the context of a single BufferLedger
- * are lockless in nature and can be leveraged by multiple threads. Operations that cross the context of two ledgers
- * will acquire a lock on the AllocationManager instance. Important note, there is one AllocationManager per
- * UnsafeDirectLittleEndian buffer allocation. As such, there will be thousands of these in a typical query. The
+ * Threading: AllocationManager manages thread-safety internally. Operations
+ * within the context of a single BufferLedger are lockless in nature and can be
+ * leveraged by multiple threads. Operations that cross the context of two
+ * ledgers will acquire a lock on the AllocationManager instance. Important
+ * note, there is one AllocationManager per UnsafeDirectLittleEndian buffer
+ * allocation. As such, there will be thousands of these in a typical query. The
* contention of acquiring a lock on AllocationManager should be very low.
- *
*/
+
public class AllocationManager {
- // private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationManager.class);
private static final AtomicLong MANAGER_ID_GENERATOR = new AtomicLong(0);
private static final AtomicLong LEDGER_ID_GENERATOR = new AtomicLong(0);
@@ -87,11 +89,13 @@ public class AllocationManager {
}
/**
- * Associate the existing underlying buffer with a new allocator. This will increase the reference count to the
- * provided ledger by 1.
+ * Associate the existing underlying buffer with a new allocator. This will
+ * increase the reference count to the provided ledger by 1.
+ *
* @param allocator
* The target allocator to associate this buffer with.
- * @return The Ledger (new or existing) that associates the underlying buffer to this new ledger.
+ * @return The Ledger (new or existing) that associates the underlying buffer
+ * to this new ledger.
*/
BufferLedger associate(final BaseAllocator allocator) {
return associate(allocator, true);
@@ -114,7 +118,6 @@ public class AllocationManager {
}
return ledger;
}
-
}
try (AutoCloseableLock write = writeLock.open()) {
// we have to recheck existing ledger since a second reader => writer could be competing with us.
@@ -138,11 +141,16 @@ public class AllocationManager {
}
}
+ public static int chunkSize() {
+ return INNER_ALLOCATOR.getChunkSize();
+ }
/**
- * The way that a particular BufferLedger communicates back to the AllocationManager that it now longer needs to hold
- * a reference to particular piece of memory.
+ * The way that a particular BufferLedger communicates back to the
+ * AllocationManager that it now longer needs to hold a reference to
+ * particular piece of memory.
*/
+
private class ReleaseListener {
private final BufferAllocator allocator;
@@ -154,6 +162,7 @@ public class AllocationManager {
/**
* Can only be called when you already hold the writeLock.
*/
+
public void release() {
allocator.assertOpen();
@@ -180,16 +189,17 @@ public class AllocationManager {
throw new IllegalStateException("The final removal of a ledger should be connected to the owning ledger.");
}
}
-
-
}
}
/**
- * The reference manager that binds an allocator manager to a particular BaseAllocator. Also responsible for creating
- * a set of DrillBufs that share a common fate and set of reference counts.
- * As with AllocationManager, the only reason this is public is due to DrillBuf being in io.netty.buffer package.
+ * The reference manager that binds an allocator manager to a particular
+ * BaseAllocator. Also responsible for creating a set of DrillBufs that share
+ * a common fate and set of reference counts. As with AllocationManager, the
+ * only reason this is public is due to DrillBuf being in io.netty.buffer
+ * package.
*/
+
public class BufferLedger {
private final IdentityHashMap<DrillBuf, Object> buffers =
@@ -294,7 +304,6 @@ public class AllocationManager {
}
}
}
-
}
private void inc() {
@@ -302,9 +311,11 @@ public class AllocationManager {
}
/**
- * Decrement the ledger's reference count. If the ledger is decremented to zero, this ledger should release its
- * ownership back to the AllocationManager
+ * Decrement the ledger's reference count. If the ledger is decremented to
+ * zero, this ledger should release its ownership back to the
+ * AllocationManager
*/
+
public int decrement(int decrement) {
allocator.assertOpen();
@@ -321,29 +332,36 @@ public class AllocationManager {
}
/**
- * Returns the ledger associated with a particular BufferAllocator. If the BufferAllocator doesn't currently have a
- * ledger associated with this AllocationManager, a new one is created. This is placed on BufferLedger rather than
- * AllocationManager directly because DrillBufs don't have access to AllocationManager and they are the ones
- * responsible for exposing the ability to associate multiple allocators with a particular piece of underlying
- * memory. Note that this will increment the reference count of this ledger by one to ensure the ledger isn't
- * destroyed before use.
+ * Returns the ledger associated with a particular BufferAllocator. If the
+ * BufferAllocator doesn't currently have a ledger associated with this
+ * AllocationManager, a new one is created. This is placed on BufferLedger
+ * rather than AllocationManager directly because DrillBufs don't have
+ * access to AllocationManager and they are the ones responsible for
+ * exposing the ability to associate multiple allocators with a particular
+ * piece of underlying memory. Note that this will increment the reference
+ * count of this ledger by one to ensure the ledger isn't destroyed before
+ * use.
*
* @param allocator
* @return
*/
+
public BufferLedger getLedgerForAllocator(BufferAllocator allocator) {
return associate((BaseAllocator) allocator);
}
/**
- * Create a new DrillBuf associated with this AllocationManager and memory. Does not impact reference count.
- * Typically used for slicing.
+ * Create a new DrillBuf associated with this AllocationManager and memory.
+ * Does not impact reference count. Typically used for slicing.
+ *
* @param offset
* The offset in bytes to start this new DrillBuf.
* @param length
* The length in bytes that this DrillBuf will provide access to.
- * @return A new DrillBuf that shares references with all DrillBufs associated with this BufferLedger
+ * @return A new DrillBuf that shares references with all DrillBufs
+ * associated with this BufferLedger
*/
+
public DrillBuf newDrillBuf(int offset, int length) {
allocator.assertOpen();
return newDrillBuf(offset, length, null);
@@ -394,6 +412,7 @@ public class AllocationManager {
*
* @return Size in bytes
*/
+
public int getSize() {
return size;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/vector/src/main/codegen/templates/ColumnAccessors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/ColumnAccessors.java b/exec/vector/src/main/codegen/templates/ColumnAccessors.java
index f1fbf2f..33b12be 100644
--- a/exec/vector/src/main/codegen/templates/ColumnAccessors.java
+++ b/exec/vector/src/main/codegen/templates/ColumnAccessors.java
@@ -19,145 +19,131 @@
<@pp.dropOutputFile />
<@pp.changeOutputFile name="/org/apache/drill/exec/vector/accessor/ColumnAccessors.java" />
<#include "/@includes/license.ftl" />
-<#macro getType label>
+<#macro getType drillType label>
@Override
public ValueType valueType() {
<#if label == "Int">
return ValueType.INTEGER;
+ <#elseif drillType == "VarChar" || drillType == "Var16Char">
+ return ValueType.STRING;
<#else>
return ValueType.${label?upper_case};
</#if>
}
</#macro>
-<#macro bindReader prefix drillType>
+<#macro bindReader vectorPrefix drillType isArray >
<#if drillType = "Decimal9" || drillType == "Decimal18">
- private MaterializedField field;
+ private MajorType type;
</#if>
- private ${prefix}${drillType}Vector.Accessor accessor;
+ private ${vectorPrefix}${drillType}Vector.Accessor accessor;
@Override
- public void bind(RowIndex vectorIndex, ValueVector vector) {
- bind(vectorIndex);
+ public void bindVector(ValueVector vector) {
<#if drillType = "Decimal9" || drillType == "Decimal18">
- field = vector.getField();
+ type = vector.getField().getType();
</#if>
- accessor = ((${prefix}${drillType}Vector) vector).getAccessor();
+ accessor = ((${vectorPrefix}${drillType}Vector) vector).getAccessor();
}
<#if drillType = "Decimal9" || drillType == "Decimal18">
@Override
- public void bind(RowIndex vectorIndex, MaterializedField field, VectorAccessor va) {
- bind(vectorIndex, field, va);
- this.field = field;
+ public void bindVector(MajorType type, VectorAccessor va) {
+ super.bindVector(type, va);
+ this.type = type;
}
</#if>
- private ${prefix}${drillType}Vector.Accessor accessor() {
+ private ${vectorPrefix}${drillType}Vector.Accessor accessor() {
if (vectorAccessor == null) {
return accessor;
} else {
- return ((${prefix}${drillType}Vector) vectorAccessor.vector()).getAccessor();
+ return ((${vectorPrefix}${drillType}Vector) vectorAccessor.vector()).getAccessor();
}
}
</#macro>
<#macro get drillType accessorType label isArray>
@Override
public ${accessorType} get${label}(<#if isArray>int index</#if>) {
+ <#assign getObject ="getObject"/>
<#if isArray>
- <#assign index=", index"/>
- <#assign getObject="getSingleObject">
+ <#assign indexVar = "index"/>
<#else>
- <#assign index=""/>
- <#assign getObject="getObject">
+ <#assign indexVar = ""/>
</#if>
- <#if drillType == "VarChar">
- return new String(accessor().get(vectorIndex.index()${index}), Charsets.UTF_8);
- <#elseif drillType == "Var16Char">
- return new String(accessor().get(vectorIndex.index()${index}), Charsets.UTF_16);
- <#elseif drillType == "VarBinary">
- return accessor().get(vectorIndex.index()${index});
+ <#if drillType == "VarChar" || drillType == "Var16Char" || drillType == "VarBinary">
+ return accessor().get(vectorIndex.vectorIndex(${indexVar}));
<#elseif drillType == "Decimal9" || drillType == "Decimal18">
return DecimalUtility.getBigDecimalFromPrimitiveTypes(
- accessor().get(vectorIndex.index()${index}),
- field.getScale(),
- field.getPrecision());
+ accessor().get(vectorIndex.vectorIndex(${indexVar})),
+ type.getScale(),
+ type.getPrecision());
<#elseif accessorType == "BigDecimal" || accessorType == "Period">
- return accessor().${getObject}(vectorIndex.index()${index});
+ return accessor().${getObject}(vectorIndex.vectorIndex(${indexVar}));
+ <#elseif drillType == "UInt1">
+ return ((int) accessor().get(vectorIndex.vectorIndex(${indexVar}))) & 0xFF;
<#else>
- return accessor().get(vectorIndex.index()${index});
+ return accessor().get(vectorIndex.vectorIndex(${indexVar}));
</#if>
}
-</#macro>
-<#macro bindWriter prefix drillType>
- <#if drillType = "Decimal9" || drillType == "Decimal18">
- private MaterializedField field;
- </#if>
- private ${prefix}${drillType}Vector.Mutator mutator;
+ <#if drillType == "VarChar">
@Override
- public void bind(RowIndex vectorIndex, ValueVector vector) {
- bind(vectorIndex);
- <#if drillType = "Decimal9" || drillType == "Decimal18">
- field = vector.getField();
- </#if>
- this.mutator = ((${prefix}${drillType}Vector) vector).getMutator();
+ public String getString(<#if isArray>int index</#if>) {
+ return new String(getBytes(${indexVar}), Charsets.UTF_8);
}
-</#macro>
-<#macro set drillType accessorType label nullable verb>
- @Override
- public void set${label}(${accessorType} value) {
- <#if drillType == "VarChar">
- byte bytes[] = value.getBytes(Charsets.UTF_8);
- mutator.${verb}Safe(vectorIndex.index(), bytes, 0, bytes.length);
<#elseif drillType == "Var16Char">
- byte bytes[] = value.getBytes(Charsets.UTF_16);
- mutator.${verb}Safe(vectorIndex.index(), bytes, 0, bytes.length);
- <#elseif drillType == "VarBinary">
- mutator.${verb}Safe(vectorIndex.index(), value, 0, value.length);
- <#elseif drillType == "Decimal9">
- mutator.${verb}Safe(vectorIndex.index(),
- DecimalUtility.getDecimal9FromBigDecimal(value,
- field.getScale(), field.getPrecision()));
- <#elseif drillType == "Decimal18">
- mutator.${verb}Safe(vectorIndex.index(),
- DecimalUtility.getDecimal18FromBigDecimal(value,
- field.getScale(), field.getPrecision()));
- <#elseif drillType == "IntervalYear">
- mutator.${verb}Safe(vectorIndex.index(), value.getYears() * 12 + value.getMonths());
- <#elseif drillType == "IntervalDay">
- mutator.${verb}Safe(vectorIndex.index(),<#if nullable> 1,</#if>
- value.getDays(),
- ((value.getHours() * 60 + value.getMinutes()) * 60 +
- value.getSeconds()) * 1000 + value.getMillis());
- <#elseif drillType == "Interval">
- mutator.${verb}Safe(vectorIndex.index(),<#if nullable> 1,</#if>
- value.getYears() * 12 + value.getMonths(),
- value.getDays(),
- ((value.getHours() * 60 + value.getMinutes()) * 60 +
- value.getSeconds()) * 1000 + value.getMillis());
+
+ @Override
+ public String getString(<#if isArray>int index</#if>) {
+ return new String(getBytes(${indexVar}), Charsets.UTF_16);
+ }
+ </#if>
+</#macro>
+<#macro build types vectorType accessorType>
+ <#if vectorType == "Repeated">
+ <#assign fnPrefix = "Array" />
+ <#assign classType = "Element" />
<#else>
- mutator.${verb}Safe(vectorIndex.index(), <#if cast=="set">(${javaType}) </#if>value);
+ <#assign fnPrefix = vectorType />
+ <#assign classType = "Scalar" />
</#if>
- }
+ <#if vectorType == "Required">
+ <#assign vectorPrefix = "" />
+ <#else>
+ <#assign vectorPrefix = vectorType />
+ </#if>
+ public static void define${fnPrefix}${accessorType}s(
+ Class<? extends Base${classType}${accessorType}> ${accessorType?lower_case}s[]) {
+ <#list types as type>
+ <#list type.minor as minor>
+ <#assign drillType=minor.class>
+ <#assign notyet=minor.accessorDisabled!type.accessorDisabled!false>
+ <#if ! notyet>
+ <#assign typeEnum=drillType?upper_case>
+ ${accessorType?lower_case}s[MinorType.${typeEnum}.ordinal()] = ${vectorPrefix}${drillType}Column${accessorType}.class;
+ </#if>
+ </#list>
+ </#list>
+ }
</#macro>
package org.apache.drill.exec.vector.accessor;
import java.math.BigDecimal;
-import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.vector.*;
-import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.util.DecimalUtility;
-import org.apache.drill.exec.vector.accessor.impl.AbstractColumnReader;
-import org.apache.drill.exec.vector.accessor.impl.AbstractColumnWriter;
-import org.apache.drill.exec.vector.complex.BaseRepeatedValueVector;
-import org.apache.drill.exec.vector.accessor.impl.AbstractArrayReader;
-import org.apache.drill.exec.vector.accessor.impl.AbstractArrayWriter;
-import org.apache.drill.exec.vector.accessor.impl.AbstractColumnReader.VectorAccessor;
+import org.apache.drill.exec.vector.accessor.reader.BaseScalarReader;
+import org.apache.drill.exec.vector.accessor.reader.BaseElementReader;
+import org.apache.drill.exec.vector.accessor.reader.VectorAccessor;
+import org.apache.drill.exec.vector.accessor.writer.BaseScalarWriter;
+import org.apache.drill.exec.vector.accessor.writer.AbstractFixedWidthWriter.BaseFixedWidthWriter;
+import org.apache.drill.exec.vector.accessor.writer.BaseVarWidthWriter;
import com.google.common.base.Charsets;
+
import org.joda.time.Period;
/**
@@ -191,141 +177,176 @@ public class ColumnAccessors {
<#if accessorType=="BigDecimal">
<#assign label="Decimal">
</#if>
+ <#if drillType == "VarChar" || drillType == "Var16Char">
+ <#assign accessorType = "byte[]">
+ <#assign label = "Bytes">
+ </#if>
<#if ! notyet>
//------------------------------------------------------------------------
// ${drillType} readers and writers
- public static class ${drillType}ColumnReader extends AbstractColumnReader {
+ public static class ${drillType}ColumnReader extends BaseScalarReader {
- <@bindReader "" drillType />
+ <@bindReader "" drillType false />
- <@getType label />
+ <@getType drillType label />
<@get drillType accessorType label false/>
}
- public static class Nullable${drillType}ColumnReader extends AbstractColumnReader {
+ public static class Nullable${drillType}ColumnReader extends BaseScalarReader {
- <@bindReader "Nullable" drillType />
+ <@bindReader "Nullable" drillType false />
- <@getType label />
+ <@getType drillType label />
@Override
public boolean isNull() {
- return accessor().isNull(vectorIndex.index());
- }
-
- <@get drillType accessorType label false/>
- }
-
- public static class Repeated${drillType}ColumnReader extends AbstractArrayReader {
-
- <@bindReader "Repeated" drillType />
-
- <@getType label />
-
- @Override
- public int size() {
- return accessor().getInnerValueCountAt(vectorIndex.index());
+ return accessor().isNull(vectorIndex.vectorIndex());
}
- <@get drillType accessorType label true/>
+ <@get drillType accessorType label false />
}
- public static class ${drillType}ColumnWriter extends AbstractColumnWriter {
+ public static class Repeated${drillType}ColumnReader extends BaseElementReader {
- <@bindWriter "" drillType />
+ <@bindReader "" drillType true />
- <@getType label />
+ <@getType drillType label />
- <@set drillType accessorType label false "set" />
+ <@get drillType accessorType label true />
}
- public static class Nullable${drillType}ColumnWriter extends AbstractColumnWriter {
-
- <@bindWriter "Nullable" drillType />
+ <#assign varWidth = drillType == "VarChar" || drillType == "Var16Char" || drillType == "VarBinary" />
+ <#if varWidth>
+ public static class ${drillType}ColumnWriter extends BaseVarWidthWriter {
+ <#else>
+ public static class ${drillType}ColumnWriter extends BaseFixedWidthWriter {
+ <#if drillType = "Decimal9" || drillType == "Decimal18" ||
+ drillType == "Decimal28Sparse" || drillType == "Decimal38Sparse">
+ private MajorType type;
+ </#if>
+ private static final int VALUE_WIDTH = ${drillType}Vector.VALUE_WIDTH;
+ </#if>
+ private final ${drillType}Vector vector;
+
+ public ${drillType}ColumnWriter(final ValueVector vector) {
+ <#if varWidth>
+ super(((${drillType}Vector) vector).getOffsetVector());
+ <#else>
+ <#if drillType = "Decimal9" || drillType == "Decimal18" ||
+ drillType == "Decimal28Sparse" || drillType == "Decimal38Sparse">
+ type = vector.getField().getType();
+ </#if>
+ </#if>
+ this.vector = (${drillType}Vector) vector;
+ }
- <@getType label />
+ @Override public BaseDataValueVector vector() { return vector; }
+
+ <#if ! varWidth>
+ @Override public int width() { return VALUE_WIDTH; }
+
+ </#if>
+ <@getType drillType label />
+ <#if accessorType == "byte[]">
+ <#assign args = ", int len">
+ <#else>
+ <#assign args = "">
+ </#if>
+ <#if javaType == "char">
+ <#assign putType = "short" />
+ <#assign doCast = true />
+ <#else>
+ <#assign putType = javaType />
+ <#assign doCast = (cast == "set") />
+ </#if>
+ <#if ! varWidth>
+ </#if>
@Override
- public void setNull() {
- mutator.setNull(vectorIndex.index());
+ public final void set${label}(final ${accessorType} value${args}) {
+ <#-- Must compute the write offset first; can't be inline because the
+ writeOffset() function has a side effect of possibly changing the buffer
+ address (bufAddr). -->
+ <#if varWidth>
+ final int offset = writeIndex(len);
+ <#else>
+ final int writeIndex = writeIndex();
+ <#assign putAddr = "writeIndex * VALUE_WIDTH">
+ </#if>
+ <#if varWidth>
+ drillBuf.unsafeCopyMemory(value, 0, offset, len);
+ offsetsWriter.setNextOffset(offset + len);
+ <#elseif drillType == "Decimal9">
+ drillBuf.unsafePutInt(${putAddr},
+ DecimalUtility.getDecimal9FromBigDecimal(value,
+ type.getScale(), type.getPrecision()));
+ <#elseif drillType == "Decimal18">
+ drillBuf.unsafePutLong(${putAddr},
+ DecimalUtility.getDecimal18FromBigDecimal(value,
+ type.getScale(), type.getPrecision()));
+ <#elseif drillType == "Decimal38Sparse">
+ <#-- Hard to optimize this case. Just use the available tools. -->
+ DecimalUtility.getSparseFromBigDecimal(value, vector.getBuffer(), writeIndex * VALUE_WIDTH,
+ type.getScale(), type.getPrecision(), 6);
+ <#elseif drillType == "Decimal28Sparse">
+ <#-- Hard to optimize this case. Just use the available tools. -->
+ DecimalUtility.getSparseFromBigDecimal(value, vector.getBuffer(), writeIndex * VALUE_WIDTH,
+ type.getScale(), type.getPrecision(), 5);
+ <#elseif drillType == "IntervalYear">
+ drillBuf.unsafePutInt(${putAddr},
+ value.getYears() * 12 + value.getMonths());
+ <#elseif drillType == "IntervalDay">
+ final int offset = ${putAddr};
+ drillBuf.unsafePutInt(offset, value.getDays());
+ drillBuf.unsafePutInt(offset + 4, periodToMillis(value));
+ <#elseif drillType == "Interval">
+ final int offset = ${putAddr};
+ drillBuf.unsafePutInt(offset, value.getYears() * 12 + value.getMonths());
+ drillBuf.unsafePutInt(offset + 4, value.getDays());
+ drillBuf.unsafePutInt(offset + 8, periodToMillis(value));
+ <#elseif drillType == "Float4">
+ drillBuf.unsafePutInt(${putAddr}, Float.floatToRawIntBits((float) value));
+ <#elseif drillType == "Float8">
+ drillBuf.unsafePutLong(${putAddr}, Double.doubleToRawLongBits(value));
+ <#else>
+ drillBuf.unsafePut${putType?cap_first}(${putAddr}, <#if doCast>(${putType}) </#if>value);
+ </#if>
+ vectorIndex.nextElement();
}
-
- <@set drillType accessorType label true "set" />
- }
-
- public static class Repeated${drillType}ColumnWriter extends AbstractArrayWriter {
-
- <@bindWriter "Repeated" drillType />
-
- <@getType label />
-
- protected BaseRepeatedValueVector.BaseRepeatedMutator mutator() {
- return mutator;
+ <#if drillType == "VarChar">
+
+ @Override
+ public final void setString(String value) {
+ final byte bytes[] = value.getBytes(Charsets.UTF_8);
+ setBytes(bytes, bytes.length);
+ }
+ <#elseif drillType == "Var16Char">
+
+ @Override
+ public final void setString(String value) {
+ final byte bytes[] = value.getBytes(Charsets.UTF_16);
+ setBytes(bytes, bytes.length);
}
-
- <@set drillType accessorType label false "add" />
- }
-
- </#if>
- </#list>
-</#list>
- public static void defineReaders(
- Class<? extends AbstractColumnReader> readers[][]) {
-<#list vv.types as type>
- <#list type.minor as minor>
- <#assign drillType=minor.class>
- <#assign notyet=minor.accessorDisabled!type.accessorDisabled!false>
- <#if ! notyet>
- <#assign typeEnum=drillType?upper_case>
- readers[MinorType.${typeEnum}.ordinal()][DataMode.REQUIRED.ordinal()] = ${drillType}ColumnReader.class;
- readers[MinorType.${typeEnum}.ordinal()][DataMode.OPTIONAL.ordinal()] = Nullable${drillType}ColumnReader.class;
</#if>
- </#list>
-</#list>
}
- public static void defineWriters(
- Class<? extends AbstractColumnWriter> writers[][]) {
-<#list vv.types as type>
- <#list type.minor as minor>
- <#assign drillType=minor.class>
- <#assign notyet=minor.accessorDisabled!type.accessorDisabled!false>
- <#if ! notyet>
- <#assign typeEnum=drillType?upper_case>
- writers[MinorType.${typeEnum}.ordinal()][DataMode.REQUIRED.ordinal()] = ${drillType}ColumnWriter.class;
- writers[MinorType.${typeEnum}.ordinal()][DataMode.OPTIONAL.ordinal()] = Nullable${drillType}ColumnWriter.class;
</#if>
</#list>
</#list>
+ public static int periodToMillis(Period value) {
+ return ((value.getHours() * 60 +
+ value.getMinutes()) * 60 +
+ value.getSeconds()) * 1000 +
+ value.getMillis();
}
- public static void defineArrayReaders(
- Class<? extends AbstractArrayReader> readers[]) {
-<#list vv.types as type>
- <#list type.minor as minor>
- <#assign drillType=minor.class>
- <#assign notyet=minor.accessorDisabled!type.accessorDisabled!false>
- <#if ! notyet>
- <#assign typeEnum=drillType?upper_case>
- readers[MinorType.${typeEnum}.ordinal()] = Repeated${drillType}ColumnReader.class;
- </#if>
- </#list>
-</#list>
- }
+<@build vv.types "Required" "Reader" />
- public static void defineArrayWriters(
- Class<? extends AbstractArrayWriter> writers[]) {
-<#list vv.types as type>
- <#list type.minor as minor>
- <#assign drillType=minor.class>
- <#assign notyet=minor.accessorDisabled!type.accessorDisabled!false>
- <#if ! notyet>
- <#assign typeEnum=drillType?upper_case>
- writers[MinorType.${typeEnum}.ordinal()] = Repeated${drillType}ColumnWriter.class;
- </#if>
- </#list>
-</#list>
- }
+<@build vv.types "Nullable" "Reader" />
+
+<@build vv.types "Repeated" "Reader" />
+
+<@build vv.types "Required" "Writer" />
}
http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
index 51938a3..1f6a008 100644
--- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -22,8 +22,8 @@
<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
<#if type.major == "Fixed">
-<@pp.changeOutputFile name="/org/apache/drill/exec/vector/${minor.class}Vector.java" />
-<#include "/@includes/license.ftl" />
+ <@pp.changeOutputFile name="/org/apache/drill/exec/vector/${minor.class}Vector.java" />
+ <#include "/@includes/license.ftl" />
package org.apache.drill.exec.vector;
@@ -31,15 +31,18 @@ package org.apache.drill.exec.vector;
import org.apache.drill.exec.util.DecimalUtility;
/**
- * ${minor.class} implements a vector of fixed width values. Elements in the vector are accessed
- * by position, starting from the logical start of the vector. Values should be pushed onto the
- * vector sequentially, but may be accessed randomly.
+ * ${minor.class} implements a vector of fixed width values. Elements in the
+ * vector are accessed by position, starting from the logical start of the
+ * vector. Values should be pushed onto the vector sequentially, but may be
+ * accessed randomly.
* <ul>
- * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width}) byte<#if type.width != 1>s</#if>.</li>
+ * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width})
+ * byte<#if type.width != 1>s</#if>.</li>
* <li>The equivalent Java primitive is '${minor.javaType!type.javaType}'.</li>
* </ul>
*
- * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
+ * NB: this class is automatically generated from ${.template_name} and
+ * ValueVectorTypes.tdd using FreeMarker.
*/
public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
@@ -151,13 +154,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
/**
- * Allocate a new buffer that supports setting at least the provided number of values. May actually be sized bigger
- * depending on underlying buffer rounding size. Must be called prior to using the ValueVector.
+ * Allocate a new buffer that supports setting at least the provided number of
+ * values. May actually be sized bigger depending on underlying buffer
+ * rounding size. Must be called prior to using the ValueVector.
*
- * Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width.
+ * Note that the maximum number of values a vector can allocate is
+ * Integer.MAX_VALUE / value width.
*
* @param valueCount
- * @throws OutOfMemoryException if it can't allocate the new buffer
+ * @throws OutOfMemoryException
+ * if it can't allocate the new buffer
*/
@Override
public void allocateNew(final int valueCount) {
@@ -211,18 +217,24 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
}
+ reallocRaw((int) newAllocationSize);
+ final int halfNewCapacity = data.capacity() / 2;
+ data.setZero(halfNewCapacity, halfNewCapacity);
+ }
+
+ @Override
+ public DrillBuf reallocRaw(int newAllocationSize) {
logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", field, allocationSizeInBytes, newAllocationSize);
if (newAllocationSize == 0) {
throw new IllegalStateException("Attempt to reAlloc a zero-sized vector");
}
- final DrillBuf newBuf = allocator.buffer((int)newAllocationSize);
+ final DrillBuf newBuf = allocator.buffer(newAllocationSize);
newBuf.setBytes(0, data, 0, data.capacity());
- final int halfNewCapacity = newBuf.capacity() / 2;
- newBuf.setZero(halfNewCapacity, halfNewCapacity);
newBuf.writerIndex(data.writerIndex());
data.release(1);
data = newBuf;
- allocationSizeInBytes = (int)newAllocationSize;
+ allocationSizeInBytes = newAllocationSize;
+ return newBuf;
}
/**
@@ -337,7 +349,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
@Override
public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
- ((${minor.class}Vector) from).data.getBytes(fromIndex * ${type.width}, data, toIndex * ${type.width}, ${type.width});
+ ((${minor.class}Vector) from).data.getBytes(fromIndex * VALUE_WIDTH, data, toIndex * VALUE_WIDTH, VALUE_WIDTH);
}
public void decrementAllocationMonitor() {
@@ -423,7 +435,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
-
return(new StringBuilder().
append(years).append(yearString).
append(months).append(monthString).
@@ -621,26 +632,31 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
/**
- * ${minor.class}.Mutator implements a mutable vector of fixed width values. Elements in the
- * vector are accessed by position from the logical start of the vector. Values should be pushed
- * onto the vector sequentially, but may be randomly accessed.
+ * ${minor.class}.Mutator implements a mutable vector of fixed width values.
+ * Elements in the vector are accessed by position from the logical start of
+ * the vector. Values should be pushed onto the vector sequentially, but may
+ * be randomly accessed.
* <ul>
- * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width}) byte(s).</li>
+ * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width})
+ * byte(s).</li>
* <li>The equivalent Java primitive is '${minor.javaType!type.javaType}'</li>
* </ul>
*
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using
+ * FreeMarker.
*/
public final class Mutator extends BaseDataValueVector.BaseMutator {
private Mutator() {};
/**
- * Set the element at the given index to the given value. Note that widths smaller than
- * 32 bits are handled by the DrillBuf interface.
+ * Set the element at the given index to the given value. Note that widths
+ * smaller than 32 bits are handled by the DrillBuf interface.
*
- * @param index position of the bit to set
- * @param value value to set
+ * @param index
+ * position of the bit to set
+ * @param value
+ * value to set
*/
<#if (type.width > 8)>
@@ -655,37 +671,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
data.setBytes(index * VALUE_WIDTH, value, 0, VALUE_WIDTH);
}
- /**
- * Set the value of a required or nullable vector. Enforces the value
- * and size limits.
- * @param index item to write
- * @param value value to set
- * @throws VectorOverflowException if the item was written, false if the index would
- * overfill the vector
- */
-
- public void setScalar(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException {
- if (index >= MAX_SCALAR_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, value);
- }
-
- /**
- * Set the value of a repeated vector. Enforces only the size limit.
- * @param index item to write
- * @param value value to set
- * @throws VectorOverflowException if the item was written, false if the index would
- * overfill the vector
- */
-
- public void setArrayItem(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException {
- if (index >= MAX_VALUE_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, value);
- }
-
<#if minor.class == "Interval">
public void set(int index, int months, int days, int milliseconds) {
final int offsetIndex = index * VALUE_WIDTH;
@@ -701,20 +686,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
set(index, months, days, milliseconds);
}
- public void setScalar(int index, int months, int days, int milliseconds) throws VectorOverflowException {
- if (index >= MAX_SCALAR_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, months, days, milliseconds);
- }
-
- public void setArrayItem(int index, int months, int days, int milliseconds) throws VectorOverflowException {
- if (index >= MAX_VALUE_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, months, days, milliseconds);
- }
-
protected void set(int index, ${minor.class}Holder holder) {
set(index, holder.months, holder.days, holder.milliseconds);
}
@@ -723,14 +694,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
setSafe(index, holder.months, holder.days, holder.milliseconds);
}
- public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException {
- setScalar(index, holder.months, holder.days, holder.milliseconds);
- }
-
- public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException {
- setArrayItem(index, holder.months, holder.days, holder.milliseconds);
- }
-
protected void set(int index, Nullable${minor.class}Holder holder) {
set(index, holder.months, holder.days, holder.milliseconds);
}
@@ -739,14 +702,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
setSafe(index, holder.months, holder.days, holder.milliseconds);
}
- public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
- setScalar(index, holder.months, holder.days, holder.milliseconds);
- }
-
- public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
- setArrayItem(index, holder.months, holder.days, holder.milliseconds);
- }
-
<#elseif minor.class == "IntervalDay">
public void set(int index, int days, int milliseconds) {
final int offsetIndex = index * VALUE_WIDTH;
@@ -761,20 +716,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
set(index, days, milliseconds);
}
- public void setScalar(int index, int days, int milliseconds) throws VectorOverflowException {
- if (index >= MAX_SCALAR_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, days, milliseconds);
- }
-
- public void setArrayItem(int index, int days, int milliseconds) throws VectorOverflowException {
- if (index >= MAX_VALUE_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, days, milliseconds);
- }
-
protected void set(int index, ${minor.class}Holder holder) {
set(index, holder.days, holder.milliseconds);
}
@@ -783,14 +724,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
setSafe(index, holder.days, holder.milliseconds);
}
- public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException {
- setScalar(index, holder.days, holder.milliseconds);
- }
-
- public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException {
- setArrayItem(index, holder.days, holder.milliseconds);
- }
-
protected void set(int index, Nullable${minor.class}Holder holder) {
set(index, holder.days, holder.milliseconds);
}
@@ -799,14 +732,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
setSafe(index, holder.days, holder.milliseconds);
}
- public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
- setScalar(index, holder.days, holder.milliseconds);
- }
-
- public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
- setArrayItem(index, holder.days, holder.milliseconds);
- }
-
<#elseif minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense">
public void setSafe(int index, int start, DrillBuf buffer) {
while(index >= getValueCapacity()) {
@@ -815,20 +740,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
set(index, start, buffer);
}
- public void setScalar(int index, int start, DrillBuf buffer) throws VectorOverflowException {
- if (index >= MAX_SCALAR_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, start, buffer);
- }
-
- public void setArrayItem(int index, int start, DrillBuf buffer) throws VectorOverflowException {
- if (index >= MAX_VALUE_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, start, buffer);
- }
-
public void set(int index, ${minor.class}Holder holder) {
set(index, holder.start, holder.buffer);
}
@@ -837,14 +748,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
setSafe(index, holder.start, holder.buffer);
}
- public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException {
- setScalar(index, holder.start, holder.buffer);
- }
-
- public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException {
- setArrayItem(index, holder.start, holder.buffer);
- }
-
void set(int index, Nullable${minor.class}Holder holder) {
set(index, holder.start, holder.buffer);
}
@@ -853,14 +756,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
setSafe(index, holder.start, holder.buffer);
}
- public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
- setScalar(index, holder.start, holder.buffer);
- }
-
- public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
- setArrayItem(index, holder.start, holder.buffer);
- }
-
<#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse">
public void set(int index, BigDecimal value) {
DecimalUtility.getSparseFromBigDecimal(value, data, index * VALUE_WIDTH,
@@ -874,20 +769,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
set(index, value);
}
- public void setScalar(int index, BigDecimal value) throws VectorOverflowException {
- if (index >= MAX_SCALAR_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, value);
- }
-
- public void setArrayItem(int index, BigDecimal value) throws VectorOverflowException {
- if (index >= MAX_VALUE_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, value);
- }
-
</#if>
public void set(int index, int start, DrillBuf buffer){
data.setBytes(index * VALUE_WIDTH, buffer, start, VALUE_WIDTH);
@@ -912,42 +793,18 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
data.set${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH, value);
}
- public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, value);
- }
-
/**
- * Set the value of a required or nullable vector. Enforces the value
- * and size limits.
+ * Set the value of a required or nullable vector. Grows the vector as needed.
+ * Does not enforce size limits; scalar fixed-width types can never overflow
+ * a vector.
* @param index item to write
- * @param value value to set
- * @throws VectorOverflowException if the item was written, false if the index would
- * overfill the vector
*/
- public void setScalar(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException {
- if (index >= MAX_SCALAR_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, value);
- }
-
- /**
- * Set the value of a repeated vector. Enforces only the size limit.
- * @param index item to write
- * @param value value to set
- * @throws VectorOverflowException if the item was written, false if the index would
- * overfill the vector
- */
-
- public void setArrayItem(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException {
- if (index >= MAX_VALUE_COUNT) {
- throw new VectorOverflowException();
+ public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+ while(index >= getValueCapacity()) {
+ reAlloc();
}
- setSafe(index, value);
+ set(index, value);
}
protected void set(int index, ${minor.class}Holder holder) {
@@ -961,20 +818,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
set(index, holder);
}
- public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException {
- if (index >= MAX_SCALAR_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, holder);
- }
-
- public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException {
- if (index >= MAX_VALUE_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, holder);
- }
-
protected void set(int index, Nullable${minor.class}Holder holder) {
data.set${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH, holder.value);
}
@@ -986,20 +829,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
set(index, holder);
}
- public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
- if (index >= MAX_SCALAR_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, holder);
- }
-
- public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
- if (index >= MAX_VALUE_COUNT) {
- throw new VectorOverflowException();
- }
- setSafe(index, holder);
- }
-
@Override
public void generateTestData(int size) {
setValueCount(size);
@@ -1028,30 +857,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
</#if> <#-- type.width -->
- /**
- * Backfill missing offsets from the given last written position to the
- * given current write position. Used by the "new" size-safe column
- * writers to allow skipping values. The <tt>set()</tt> and <tt>setSafe()</tt>
- * <b>do not</b> fill empties. See DRILL-5529 and DRILL-5530.
- * @param lastWrite the position of the last valid write: the offset
- * to be copied forward
- * @param index the current write position filling occurs up to,
- * but not including, this position
- * @throws VectorOverflowException if the item was written, false if the index would
- * overfill the vector
- */
-
- public void fillEmptiesBounded(int lastWrite, int index)
- throws VectorOverflowException {
- <#if type.width <= 8>
- for (int i = lastWrite + 1; i <= index; i++) {
- setSafe(i, <#if (type.width >= 4)>(${minor.javaType!type.javaType})</#if> 0);
- }
- <#else>
- throw new UnsupportedOperationException("Cannot zero-fill ${minor.class} vectors.");
- </#if>
- }
-
@Override
public void setValueCount(int valueCount) {
final int currentValueCapacity = getValueCapacity();
http://git-wip-us.apache.org/repos/asf/drill/blob/40de8ca4/exec/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
index 4f3eb17..fdb0200 100644
--- a/exec/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -47,7 +47,7 @@ package org.apache.drill.exec.vector;
*
* NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
*/
-@SuppressWarnings("unused")
+
public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
@@ -180,6 +180,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
}
@Override
+ public DrillBuf reallocRaw(int newAllocationSize) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public void collectLedgers(Set<BufferLedger> ledgers) {
bits.collectLedgers(ledgers);
values.collectLedgers(ledgers);
@@ -304,7 +309,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
}
private class TransferImpl implements TransferPair {
- Nullable${minor.class}Vector to;
+ private final Nullable${minor.class}Vector to;
public TransferImpl(MaterializedField field, BufferAllocator allocator){
to = new Nullable${minor.class}Vector(field, allocator);
@@ -336,12 +341,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type
}
@Override
- public Accessor getAccessor(){
+ public Accessor getAccessor() {
return accessor;
}
@Override
- public Mutator getMutator(){
+ public Mutator getMutator() {
return mutator;
}
@@ -548,16 +553,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
lastSet = index;
}
- public void setScalar(int index, byte[] value, int start, int length) throws VectorOverflowException {
- if (index > lastSet + 1) {
- fillEmpties(index); // Filling empties cannot overflow the vector
- }
- values.getMutator().setScalar(index, value, start, length);
- bits.getMutator().setSafe(index, 1);
- setCount++;
- lastSet = index;
- }
-
public void setSafe(int index, ByteBuffer value, int start, int length) {
if (index > lastSet + 1) {
fillEmpties(index);
@@ -569,17 +564,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
lastSet = index;
}
- public void setScalar(int index, DrillBuf value, int start, int length) throws VectorOverflowException {
- if (index > lastSet + 1) {
- fillEmpties(index); // Filling empties cannot overflow the vector
- }
-
- values.getMutator().setScalar(index, value, start, length);
- bits.getMutator().setSafe(index, 1);
- setCount++;
- lastSet = index;
- }
-
</#if>
public void setNull(int index) {
bits.getMutator().setSafe(index, 0);
@@ -593,10 +577,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
values.getMutator().set(index, holder);
}
- public void setNullBounded(int index) throws VectorOverflowException {
- bits.getMutator().setScalar(index, 0);
- }
-
public void set(int index, Nullable${minor.class}Holder holder) {
final ${valuesName}.Mutator valuesMutator = values.getMutator();
<#if type.major == "VarLen">
@@ -648,18 +628,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
setCount++;
<#if type.major == "VarLen">lastSet = index;</#if>
- }
-
- public void setScalar(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) throws VectorOverflowException {
- <#if type.major == "VarLen">
- if (index > lastSet + 1) {
- fillEmpties(index);
- }
- </#if>
- values.getMutator().setScalar(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
- bits.getMutator().setSafe(index, isSet);
- setCount++;
- <#if type.major == "VarLen">lastSet = index;</#if>
}
public void setSafe(int index, Nullable${minor.class}Holder value) {
@@ -674,18 +642,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#if type.major == "VarLen">lastSet = index;</#if>
}
- public void setScalar(int index, Nullable${minor.class}Holder value) throws VectorOverflowException {
- <#if type.major == "VarLen">
- if (index > lastSet + 1) {
- fillEmpties(index);
- }
- </#if>
- values.getMutator().setScalar(index, value);
- bits.getMutator().setSafe(index, value.isSet);
- setCount++;
- <#if type.major == "VarLen">lastSet = index;</#if>
- }
-
public void setSafe(int index, ${minor.class}Holder value) {
<#if type.major == "VarLen">
if (index > lastSet + 1) {
@@ -698,18 +654,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#if type.major == "VarLen">lastSet = index;</#if>
}
- public void setScalar(int index, ${minor.class}Holder value) throws VectorOverflowException {
- <#if type.major == "VarLen">
- if (index > lastSet + 1) {
- fillEmpties(index);
- }
- </#if>
- values.getMutator().setScalar(index, value);
- bits.getMutator().setSafe(index, 1);
- setCount++;
- <#if type.major == "VarLen">lastSet = index;</#if>
- }
-
<#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "Interval" || minor.class == "IntervalDay")>
public void setSafe(int index, ${minor.javaType!type.javaType} value) {
<#if type.major == "VarLen">
@@ -722,17 +666,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
setCount++;
}
- public void setScalar(int index, ${minor.javaType!type.javaType} value) throws VectorOverflowException {
- <#if type.major == "VarLen">
- if (index > lastSet + 1) {
- fillEmpties(index);
- }
- </#if>
- values.getMutator().setScalar(index, value);
- bits.getMutator().setSafe(index, 1);
- setCount++;
- }
-
</#if>
<#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse">
public void set(int index, BigDecimal value) {
@@ -747,12 +680,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
setCount++;
}
- public void setScalar(int index, BigDecimal value) throws VectorOverflowException {
- values.getMutator().setScalar(index, value);
- bits.getMutator().setSafe(index, 1);
- setCount++;
- }
-
</#if>
@Override
public void setValueCount(int valueCount) {